{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib qt5\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import time\n",
    "import matplotlib.pyplot as plt\n",
    "pd.set_option(\"display.max_rows\",999)\n",
    "Alpha=0.1 #Learning Rate\n",
    "Beta=0.9 #The deceasing rate of the future gain\n",
    "#The rate of choosing a randon action ,Epsilon\n",
    "Epsilon_start=1\n",
    "Epsilon_stop=0.01\n",
    "Decay_rate=0.00001\n",
    "LengthOfMesh=10\n",
    "TotalNumberOfMesh=LengthOfMesh*LengthOfMesh #We assume that the mesh forme is 10*10\n",
    "Actions=['Left_three','Left_four','Left_six','Right_three','Right_four','Right_six','Up_three','Up_four','Up_six']\n",
    "Obstacle=[3,5,16,21,23,27,29,31,33,34,56,57,66,67,78,81,86]\n",
    "#Only the Six_Gait can cross this type of obstacles\n",
    "Obstacle_six=[6,7,22,28,44,55,68,82,83,92]\n",
    "#Only the Six_Gait and Four_Gait can cross this type of obstacles, howerer the Four_Gait is the best\n",
    "Obstacle_four=[8,10,11,12,15,24,30,40,45,52,72,79,85]\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Caculate_coordinate(Current_state):\n",
    "    x_axis=0.5+(Current_state-1)%LengthOfMesh\n",
    "    y_axis=0.5+((Current_state-1)//LengthOfMesh)\n",
    "    return x_axis,y_axis\n",
    "def Initial_Q_Table (NumberOfStates,Actions):\n",
    "    Q_Table=np.zeros([NumberOfStates,len(Actions)])\n",
    "    Q_Table=pd.DataFrame(Q_Table,index=np.arange(1,NumberOfStates+1),columns=Actions)\n",
    "    print ('We have finished initializing the Q-table') \n",
    "    return Q_Table\n",
    "#——————————————————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "#According to the current state, we choose an action and then execute it.\n",
    "def Select_action(State,Q_Table,Action_times):\n",
    "    Epsilon=Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-Decay_rate*Action_times)\n",
    "    States_Actions=Q_Table.loc[[State],:]\n",
    "    if(np.random.rand()<Epsilon or np.all(States_Actions==[0])):\n",
    "        Execute_action=np.random.choice(Actions)#Select an action randomy\n",
    "    else:\n",
    "        Execute_action=np.array(pd.DataFrame.idxmax((Q_Table.T).loc[:,[State]]))[0]#Select the action having the maximun QValue\n",
    "    return Execute_action\n",
    "#———————————————————————————————————————————————————————————————————————————————————————————————————————————\n",
    "#According to the current state and the selected current action, return the new state and rewards\n",
    "def Output_next_state(Current_state,Execute_action):\n",
    "    if(Execute_action=='Left_three'):\n",
    "        if(Current_state%LengthOfMesh==1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_four):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        else:\n",
    "            Next_state=Current_state-1\n",
    "            Reward=1\n",
    "    elif(Execute_action=='Left_four'):\n",
    "        if(Current_state%LengthOfMesh==1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_four):\n",
    "            Next_state=Current_state-1\n",
    "            Reward=1\n",
    "        else:\n",
    "            Next_state=Current_state-1\n",
    "            Reward=0.6\n",
    "    elif(Execute_action=='Left_six'):\n",
    "        if(Current_state%LengthOfMesh==1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state-1) in Obstacle_six):\n",
    "            Next_state=Current_state-1\n",
    "            Reward=1\n",
    "        elif((Current_state-1) in Obstacle_four):\n",
    "            Next_state=Current_state-1\n",
    "            Reward=0.6\n",
    "        else:\n",
    "            Next_state=Current_state-1\n",
    "            Reward=0.2\n",
    "    \n",
    "    elif(Execute_action=='Right_three'):\n",
    "        if(Current_state==TotalNumberOfMesh-1):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(Current_state%LengthOfMesh==0):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_four):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        else:\n",
    "            Next_state=Current_state+1\n",
    "            Reward=1\n",
    "    elif(Execute_action=='Right_four'):\n",
    "        if(Current_state==TotalNumberOfMesh-1):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(Current_state%LengthOfMesh==0):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_four):\n",
    "            Next_state=Current_state+1\n",
    "            Reward=1\n",
    "        else:\n",
    "            Next_state=Current_state+1\n",
    "            Reward=0.6\n",
    "    elif(Execute_action=='Right_six'):\n",
    "        if(Current_state==TotalNumberOfMesh-1):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(Current_state%LengthOfMesh==0):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+1)in Obstacle_six):\n",
    "            Next_state=Current_state+1\n",
    "            Reward=1\n",
    "        elif((Current_state+1)in Obstacle_four):\n",
    "            Next_state=Current_state+1\n",
    "            Reward=0.6\n",
    "        else:\n",
    "            Next_state=Current_state+1\n",
    "            Reward=0.2\n",
    "    \n",
    "    elif(Execute_action=='Up_three'):\n",
    "        if(Current_state==TotalNumberOfMesh-LengthOfMesh):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(TotalNumberOfMesh+1-LengthOfMesh<=Current_state<=TotalNumberOfMesh-1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_four):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        else:\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=1\n",
    "    elif(Execute_action=='Up_four'):\n",
    "        if(Current_state==TotalNumberOfMesh-LengthOfMesh):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(TotalNumberOfMesh+1-LengthOfMesh<=Current_state<=TotalNumberOfMesh-1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_six):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_four):\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=1\n",
    "        else:\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=0.6\n",
    "    elif(Execute_action=='Up_six'):\n",
    "        if(Current_state==TotalNumberOfMesh-LengthOfMesh):\n",
    "            Next_state='End'\n",
    "            Reward=1000\n",
    "        elif(TotalNumberOfMesh+1-LengthOfMesh<=Current_state<=TotalNumberOfMesh-1):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle):\n",
    "            Next_state=Current_state\n",
    "            Reward=-20\n",
    "        elif((Current_state+10) in Obstacle_six):\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=1\n",
    "        elif((Current_state+10) in Obstacle_four):\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=0.6\n",
    "        else:\n",
    "            Next_state=Current_state+LengthOfMesh\n",
    "            Reward=0.2\n",
    "    return Next_state,Reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Plot_Mesh():\n",
    "    fig=plt.figure()\n",
    "    ax=fig.gca()\n",
    "    ax.set(xlim=[0, LengthOfMesh], ylim=[0, LengthOfMesh])\n",
    "    ax.set_xticks(np.arange(0,(LengthOfMesh+1)))\n",
    "    ax.set_yticks(np.arange(0,(LengthOfMesh+1)))\n",
    "    plt.tick_params(labelsize=30)\n",
    "    plt.xlabel('x',size=30)\n",
    "    plt.ylabel('y',size=30)\n",
    "    plt.grid()\n",
    "    fig.suptitle('original grid map (three types of obstacles)', fontsize=40)\n",
    "    for J in np.arange(len(Obstacle)):\n",
    "        plt.scatter((Caculate_coordinate(Obstacle[J]))[0],(Caculate_coordinate(Obstacle[J]))[1],marker=\"x\",color='k',s=300)\n",
    "    for J in np.arange(len(Obstacle_six)):\n",
    "        plt.scatter((Caculate_coordinate(Obstacle_six[J]))[0],(Caculate_coordinate(Obstacle_six[J]))[1],marker=\"H\",color='k',s=300)\n",
    "    for J in np.arange(len(Obstacle_four)):\n",
    "        plt.scatter((Caculate_coordinate(Obstacle_four[J]))[0],(Caculate_coordinate(Obstacle_four[J]))[1],marker=\"s\",color='k',s=300)\n",
    "    plt.show()\n",
    "# Plot_Mesh()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have finished initializing the Q-table\n",
      "Epoche=0\n",
      "Action_times=104\n",
      "Episolon=0.998971\n",
      "Epoche=1\n",
      "Action_times=114\n",
      "Episolon=0.998872\n",
      "Epoche=2\n",
      "Action_times=168\n",
      "Episolon=0.998338\n",
      "Epoche=3\n",
      "Action_times=176\n",
      "Episolon=0.998259\n",
      "Epoche=4\n",
      "Action_times=344\n",
      "Episolon=0.996600\n",
      "Epoche=5\n",
      "Action_times=786\n",
      "Episolon=0.992249\n",
      "Epoche=6\n",
      "Action_times=843\n",
      "Episolon=0.991689\n",
      "Epoche=7\n",
      "Action_times=1024\n",
      "Episolon=0.989914\n",
      "Epoche=8\n",
      "Action_times=1031\n",
      "Episolon=0.989846\n",
      "Epoche=9\n",
      "Action_times=1056\n",
      "Episolon=0.989601\n",
      "Epoche=10\n",
      "Action_times=1066\n",
      "Episolon=0.989503\n",
      "Epoche=11\n",
      "Action_times=1136\n",
      "Episolon=0.988817\n",
      "Epoche=12\n",
      "Action_times=1165\n",
      "Episolon=0.988533\n",
      "Epoche=13\n",
      "Action_times=1288\n",
      "Episolon=0.987331\n",
      "Epoche=14\n",
      "Action_times=1314\n",
      "Episolon=0.987076\n",
      "Epoche=15\n",
      "Action_times=1420\n",
      "Episolon=0.986041\n",
      "Epoche=16\n",
      "Action_times=1727\n",
      "Episolon=0.983049\n",
      "Epoche=17\n",
      "Action_times=1817\n",
      "Episolon=0.982174\n",
      "Epoche=18\n",
      "Action_times=1887\n",
      "Episolon=0.981494\n",
      "Epoche=19\n",
      "Action_times=1915\n",
      "Episolon=0.981222\n",
      "Epoche=20\n",
      "Action_times=2026\n",
      "Episolon=0.980144\n",
      "Epoche=21\n",
      "Action_times=2120\n",
      "Episolon=0.979233\n",
      "Epoche=22\n",
      "Action_times=2379\n",
      "Episolon=0.976726\n",
      "Epoche=23\n",
      "Action_times=2450\n",
      "Episolon=0.976040\n",
      "Epoche=24\n",
      "Action_times=2464\n",
      "Episolon=0.975904\n",
      "Epoche=25\n",
      "Action_times=2479\n",
      "Episolon=0.975760\n",
      "Epoche=26\n",
      "Action_times=2503\n",
      "Episolon=0.975528\n",
      "Epoche=27\n",
      "Action_times=2518\n",
      "Episolon=0.975383\n",
      "Epoche=28\n",
      "Action_times=2669\n",
      "Episolon=0.973926\n",
      "Epoche=29\n",
      "Action_times=2685\n",
      "Episolon=0.973772\n",
      "Epoche=30\n",
      "Action_times=2740\n",
      "Episolon=0.973242\n",
      "Epoche=31\n",
      "Action_times=2771\n",
      "Episolon=0.972944\n",
      "Epoche=32\n",
      "Action_times=2800\n",
      "Episolon=0.972664\n",
      "Epoche=33\n",
      "Action_times=2898\n",
      "Episolon=0.971722\n",
      "Epoche=34\n",
      "Action_times=3189\n",
      "Episolon=0.968927\n",
      "Epoche=35\n",
      "Action_times=3282\n",
      "Episolon=0.968036\n",
      "Epoche=36\n",
      "Action_times=3361\n",
      "Episolon=0.967279\n",
      "Epoche=37\n",
      "Action_times=3404\n",
      "Episolon=0.966868\n",
      "Epoche=38\n",
      "Action_times=3470\n",
      "Episolon=0.966236\n",
      "Epoche=39\n",
      "Action_times=3510\n",
      "Episolon=0.965854\n",
      "Epoche=40\n",
      "Action_times=3584\n",
      "Episolon=0.965147\n",
      "Epoche=41\n",
      "Action_times=3606\n",
      "Episolon=0.964937\n",
      "Epoche=42\n",
      "Action_times=3640\n",
      "Episolon=0.964612\n",
      "Epoche=43\n",
      "Action_times=3733\n",
      "Episolon=0.963725\n",
      "Epoche=44\n",
      "Action_times=3772\n",
      "Episolon=0.963353\n",
      "Epoche=45\n",
      "Action_times=3800\n",
      "Episolon=0.963086\n",
      "Epoche=46\n",
      "Action_times=3814\n",
      "Episolon=0.962952\n",
      "Epoche=47\n",
      "Action_times=4023\n",
      "Episolon=0.960963\n",
      "Epoche=48\n",
      "Action_times=4113\n",
      "Episolon=0.960107\n",
      "Epoche=49\n",
      "Action_times=4124\n",
      "Episolon=0.960003\n",
      "Epoche=50\n",
      "Action_times=4300\n",
      "Episolon=0.958332\n",
      "Epoche=51\n",
      "Action_times=4363\n",
      "Episolon=0.957735\n",
      "Epoche=52\n",
      "Action_times=4419\n",
      "Episolon=0.957204\n",
      "Epoche=53\n",
      "Action_times=4492\n",
      "Episolon=0.956513\n",
      "Epoche=54\n",
      "Action_times=4519\n",
      "Episolon=0.956258\n",
      "Epoche=55\n",
      "Action_times=4630\n",
      "Episolon=0.955208\n",
      "Epoche=56\n",
      "Action_times=5097\n",
      "Episolon=0.950804\n",
      "Epoche=57\n",
      "Action_times=5141\n",
      "Episolon=0.950390\n",
      "Epoche=58\n",
      "Action_times=5173\n",
      "Episolon=0.950089\n",
      "Epoche=59\n",
      "Action_times=5246\n",
      "Episolon=0.949403\n",
      "Epoche=60\n",
      "Action_times=5539\n",
      "Episolon=0.946655\n",
      "Epoche=61\n",
      "Action_times=5623\n",
      "Episolon=0.945868\n",
      "Epoche=62\n",
      "Action_times=5686\n",
      "Episolon=0.945279\n",
      "Epoche=63\n",
      "Action_times=5735\n",
      "Episolon=0.944821\n",
      "Epoche=64\n",
      "Action_times=5776\n",
      "Episolon=0.944438\n",
      "Epoche=65\n",
      "Action_times=5819\n",
      "Episolon=0.944036\n",
      "Epoche=66\n",
      "Action_times=5952\n",
      "Episolon=0.942795\n",
      "Epoche=67\n",
      "Action_times=6124\n",
      "Episolon=0.941191\n",
      "Epoche=68\n",
      "Action_times=6298\n",
      "Episolon=0.939573\n",
      "Epoche=69\n",
      "Action_times=6495\n",
      "Episolon=0.937743\n",
      "Epoche=70\n",
      "Action_times=6506\n",
      "Episolon=0.937641\n",
      "Epoche=71\n",
      "Action_times=6528\n",
      "Episolon=0.937437\n",
      "Epoche=72\n",
      "Action_times=6622\n",
      "Episolon=0.936566\n",
      "Epoche=73\n",
      "Action_times=6648\n",
      "Episolon=0.936325\n",
      "Epoche=74\n",
      "Action_times=6664\n",
      "Episolon=0.936177\n",
      "Epoche=75\n",
      "Action_times=6717\n",
      "Episolon=0.935686\n",
      "Epoche=76\n",
      "Action_times=6748\n",
      "Episolon=0.935399\n",
      "Epoche=77\n",
      "Action_times=6786\n",
      "Episolon=0.935047\n",
      "Epoche=78\n",
      "Action_times=6806\n",
      "Episolon=0.934862\n",
      "Epoche=79\n",
      "Action_times=6819\n",
      "Episolon=0.934742\n",
      "Epoche=80\n",
      "Action_times=6902\n",
      "Episolon=0.933975\n",
      "Epoche=81\n",
      "Action_times=6922\n",
      "Episolon=0.933790\n",
      "Epoche=82\n",
      "Action_times=7013\n",
      "Episolon=0.932950\n",
      "Epoche=83\n",
      "Action_times=7103\n",
      "Episolon=0.932120\n",
      "Epoche=84\n",
      "Action_times=7124\n",
      "Episolon=0.931926\n",
      "Epoche=85\n",
      "Action_times=7154\n",
      "Episolon=0.931649\n",
      "Epoche=86\n",
      "Action_times=7205\n",
      "Episolon=0.931180\n",
      "Epoche=87\n",
      "Action_times=7296\n",
      "Episolon=0.930342\n",
      "Epoche=88\n",
      "Action_times=7396\n",
      "Episolon=0.929422\n",
      "Epoche=89\n",
      "Action_times=7438\n",
      "Episolon=0.929036\n",
      "Epoche=90\n",
      "Action_times=7456\n",
      "Episolon=0.928870\n",
      "Epoche=91\n",
      "Action_times=7559\n",
      "Episolon=0.927924\n",
      "Epoche=92\n",
      "Action_times=7666\n",
      "Episolon=0.926943\n",
      "Epoche=93\n",
      "Action_times=7853\n",
      "Episolon=0.925230\n",
      "Epoche=94\n",
      "Action_times=7882\n",
      "Episolon=0.924964\n",
      "Epoche=95\n",
      "Action_times=7931\n",
      "Episolon=0.924516\n",
      "Epoche=96\n",
      "Action_times=7944\n",
      "Episolon=0.924397\n",
      "Epoche=97\n",
      "Action_times=8037\n",
      "Episolon=0.923547\n",
      "Epoche=98\n",
      "Action_times=8116\n",
      "Episolon=0.922826\n",
      "Epoche=99\n",
      "Action_times=8287\n",
      "Episolon=0.921266\n",
      "Epoche=100\n",
      "Action_times=8305\n",
      "Episolon=0.921102\n",
      "Epoche=101\n",
      "Action_times=8309\n",
      "Episolon=0.921066\n",
      "Epoche=102\n",
      "Action_times=8390\n",
      "Episolon=0.920328\n",
      "Epoche=103\n",
      "Action_times=8450\n",
      "Episolon=0.919782\n",
      "Epoche=104\n",
      "Action_times=8520\n",
      "Episolon=0.919145\n",
      "Epoche=105\n",
      "Action_times=8595\n",
      "Episolon=0.918464\n",
      "Epoche=106\n",
      "Action_times=8700\n",
      "Episolon=0.917510\n",
      "Epoche=107\n",
      "Action_times=8797\n",
      "Episolon=0.916630\n",
      "Epoche=108\n",
      "Action_times=8888\n",
      "Episolon=0.915806\n",
      "Epoche=109\n",
      "Action_times=8929\n",
      "Episolon=0.915435\n",
      "Epoche=110\n",
      "Action_times=8985\n",
      "Episolon=0.914928\n",
      "Epoche=111\n",
      "Action_times=9061\n",
      "Episolon=0.914240\n",
      "Epoche=112\n",
      "Action_times=9183\n",
      "Episolon=0.913138\n",
      "Epoche=113\n",
      "Action_times=9193\n",
      "Episolon=0.913047\n",
      "Epoche=114\n",
      "Action_times=9280\n",
      "Episolon=0.912262\n",
      "Epoche=115\n",
      "Action_times=9355\n",
      "Episolon=0.911586\n",
      "Epoche=116\n",
      "Action_times=9400\n",
      "Episolon=0.911180\n",
      "Epoche=117\n",
      "Action_times=9441\n",
      "Episolon=0.910811\n",
      "Epoche=118\n",
      "Action_times=9564\n",
      "Episolon=0.909703\n",
      "Epoche=119\n",
      "Action_times=9594\n",
      "Episolon=0.909433\n",
      "Epoche=120\n",
      "Action_times=9643\n",
      "Episolon=0.908993\n",
      "Epoche=121\n",
      "Action_times=9743\n",
      "Episolon=0.908094\n",
      "Epoche=122\n",
      "Action_times=9839\n",
      "Episolon=0.907232\n",
      "Epoche=123\n",
      "Action_times=9844\n",
      "Episolon=0.907188\n",
      "Epoche=124\n",
      "Action_times=9914\n",
      "Episolon=0.906560\n",
      "Epoche=125\n",
      "Action_times=9975\n",
      "Episolon=0.906013\n",
      "Epoche=126\n",
      "Action_times=10083\n",
      "Episolon=0.905046\n",
      "Epoche=127\n",
      "Action_times=10098\n",
      "Episolon=0.904912\n",
      "Epoche=128\n",
      "Action_times=10109\n",
      "Episolon=0.904813\n",
      "Epoche=129\n",
      "Action_times=10279\n",
      "Episolon=0.903293\n",
      "Epoche=130\n",
      "Action_times=10349\n",
      "Episolon=0.902668\n",
      "Epoche=131\n",
      "Action_times=10398\n",
      "Episolon=0.902231\n",
      "Epoche=132\n",
      "Action_times=10405\n",
      "Episolon=0.902168\n",
      "Epoche=133\n",
      "Action_times=10511\n",
      "Episolon=0.901223\n",
      "Epoche=134\n",
      "Action_times=10572\n",
      "Episolon=0.900680\n",
      "Epoche=135\n",
      "Action_times=10619\n",
      "Episolon=0.900261\n",
      "Epoche=136\n",
      "Action_times=10633\n",
      "Episolon=0.900137\n",
      "Epoche=137\n",
      "Action_times=10719\n",
      "Episolon=0.899371\n",
      "Epoche=138\n",
      "Action_times=10839\n",
      "Episolon=0.898305\n",
      "Epoche=139\n",
      "Action_times=10876\n",
      "Episolon=0.897976\n",
      "Epoche=140\n",
      "Action_times=10961\n",
      "Episolon=0.897222\n",
      "Epoche=141\n",
      "Action_times=10996\n",
      "Episolon=0.896911\n",
      "Epoche=142\n",
      "Action_times=11078\n",
      "Episolon=0.896184\n",
      "Epoche=143\n",
      "Action_times=11178\n",
      "Episolon=0.895299\n",
      "Epoche=144\n",
      "Action_times=11197\n",
      "Episolon=0.895130\n",
      "Epoche=145\n",
      "Action_times=11288\n",
      "Episolon=0.894325\n",
      "Epoche=146\n",
      "Action_times=11302\n",
      "Episolon=0.894201\n",
      "Epoche=147\n",
      "Action_times=11320\n",
      "Episolon=0.894042\n",
      "Epoche=148\n",
      "Action_times=11355\n",
      "Episolon=0.893733\n",
      "Epoche=149\n",
      "Action_times=11442\n",
      "Episolon=0.892964\n",
      "Epoche=150\n",
      "Action_times=11454\n",
      "Episolon=0.892859\n",
      "Epoche=151\n",
      "Action_times=11509\n",
      "Episolon=0.892373\n",
      "Epoche=152\n",
      "Action_times=11543\n",
      "Episolon=0.892073\n",
      "Epoche=153\n",
      "Action_times=11611\n",
      "Episolon=0.891474\n",
      "Epoche=154\n",
      "Action_times=11767\n",
      "Episolon=0.890099\n",
      "Epoche=155\n",
      "Action_times=11828\n",
      "Episolon=0.889563\n",
      "Epoche=156\n",
      "Action_times=11863\n",
      "Episolon=0.889255\n",
      "Epoche=157\n",
      "Action_times=11891\n",
      "Episolon=0.889009\n",
      "Epoche=158\n",
      "Action_times=11942\n",
      "Episolon=0.888561\n",
      "Epoche=159\n",
      "Action_times=11984\n",
      "Episolon=0.888192\n",
      "Epoche=160\n",
      "Action_times=12052\n",
      "Episolon=0.887595\n",
      "Epoche=161\n",
      "Action_times=12128\n",
      "Episolon=0.886928\n",
      "Epoche=162\n",
      "Action_times=12178\n",
      "Episolon=0.886490\n",
      "Epoche=163\n",
      "Action_times=12204\n",
      "Episolon=0.886262\n",
      "Epoche=164\n",
      "Action_times=12346\n",
      "Episolon=0.885018\n",
      "Epoche=165\n",
      "Action_times=12361\n",
      "Episolon=0.884887\n",
      "Epoche=166\n",
      "Action_times=12435\n",
      "Episolon=0.884240\n",
      "Epoche=167\n",
      "Action_times=12484\n",
      "Episolon=0.883812\n",
      "Epoche=168\n",
      "Action_times=12512\n",
      "Episolon=0.883567\n",
      "Epoche=169\n",
      "Action_times=12518\n",
      "Episolon=0.883515\n",
      "Epoche=170\n",
      "Action_times=12577\n",
      "Episolon=0.882999\n",
      "Epoche=171\n",
      "Action_times=12716\n",
      "Episolon=0.881787\n",
      "Epoche=172\n",
      "Action_times=12813\n",
      "Episolon=0.880942\n",
      "Epoche=173\n",
      "Action_times=12879\n",
      "Episolon=0.880367\n",
      "Epoche=174\n",
      "Action_times=13000\n",
      "Episolon=0.879314\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoche=175\n",
      "Action_times=13058\n",
      "Episolon=0.878810\n",
      "Epoche=176\n",
      "Action_times=13196\n",
      "Episolon=0.877612\n",
      "Epoche=177\n",
      "Action_times=13305\n",
      "Episolon=0.876667\n",
      "Epoche=178\n",
      "Action_times=13340\n",
      "Episolon=0.876364\n",
      "Epoche=179\n",
      "Action_times=13376\n",
      "Episolon=0.876052\n",
      "Epoche=180\n",
      "Action_times=13423\n",
      "Episolon=0.875645\n",
      "Epoche=181\n",
      "Action_times=13432\n",
      "Episolon=0.875567\n",
      "Epoche=182\n",
      "Action_times=13598\n",
      "Episolon=0.874131\n",
      "Epoche=183\n",
      "Action_times=13731\n",
      "Episolon=0.872983\n",
      "Epoche=184\n",
      "Action_times=13768\n",
      "Episolon=0.872664\n",
      "Epoche=185\n",
      "Action_times=13840\n",
      "Episolon=0.872043\n",
      "Epoche=186\n",
      "Action_times=13866\n",
      "Episolon=0.871819\n",
      "Epoche=187\n",
      "Action_times=13883\n",
      "Episolon=0.871672\n",
      "Epoche=188\n",
      "Action_times=13899\n",
      "Episolon=0.871534\n",
      "Epoche=189\n",
      "Action_times=13993\n",
      "Episolon=0.870725\n",
      "Epoche=190\n",
      "Action_times=14022\n",
      "Episolon=0.870475\n",
      "Epoche=191\n",
      "Action_times=14038\n",
      "Episolon=0.870338\n",
      "Epoche=192\n",
      "Action_times=14039\n",
      "Episolon=0.870329\n",
      "Epoche=193\n",
      "Action_times=14056\n",
      "Episolon=0.870183\n",
      "Epoche=194\n",
      "Action_times=14133\n",
      "Episolon=0.869521\n",
      "Epoche=195\n",
      "Action_times=14147\n",
      "Episolon=0.869400\n",
      "Epoche=196\n",
      "Action_times=14161\n",
      "Episolon=0.869280\n",
      "Epoche=197\n",
      "Action_times=14175\n",
      "Episolon=0.869160\n",
      "Epoche=198\n",
      "Action_times=14247\n",
      "Episolon=0.868541\n",
      "Epoche=199\n",
      "Action_times=14263\n",
      "Episolon=0.868404\n",
      "Epoche=200\n",
      "Action_times=14420\n",
      "Episolon=0.867057\n",
      "Epoche=201\n",
      "Action_times=14570\n",
      "Episolon=0.865773\n",
      "Epoche=202\n",
      "Action_times=14632\n",
      "Episolon=0.865242\n",
      "Epoche=203\n",
      "Action_times=14678\n",
      "Episolon=0.864849\n",
      "Epoche=204\n",
      "Action_times=14720\n",
      "Episolon=0.864490\n",
      "Epoche=205\n",
      "Action_times=14753\n",
      "Episolon=0.864208\n",
      "Epoche=206\n",
      "Action_times=14801\n",
      "Episolon=0.863798\n",
      "Epoche=207\n",
      "Action_times=14861\n",
      "Episolon=0.863286\n",
      "Epoche=208\n",
      "Action_times=14872\n",
      "Episolon=0.863192\n",
      "Epoche=209\n",
      "Action_times=14899\n",
      "Episolon=0.862962\n",
      "Epoche=210\n",
      "Action_times=14921\n",
      "Episolon=0.862774\n",
      "Epoche=211\n",
      "Action_times=14928\n",
      "Episolon=0.862715\n",
      "Epoche=212\n",
      "Action_times=14939\n",
      "Episolon=0.862621\n",
      "Epoche=213\n",
      "Action_times=14979\n",
      "Episolon=0.862280\n",
      "Epoche=214\n",
      "Action_times=15044\n",
      "Episolon=0.861726\n",
      "Epoche=215\n",
      "Action_times=15077\n",
      "Episolon=0.861445\n",
      "Epoche=216\n",
      "Action_times=15106\n",
      "Episolon=0.861198\n",
      "Epoche=217\n",
      "Action_times=15156\n",
      "Episolon=0.860773\n",
      "Epoche=218\n",
      "Action_times=15230\n",
      "Episolon=0.860143\n",
      "Epoche=219\n",
      "Action_times=15231\n",
      "Episolon=0.860135\n",
      "Epoche=220\n",
      "Action_times=15274\n",
      "Episolon=0.859769\n",
      "Epoche=221\n",
      "Action_times=15367\n",
      "Episolon=0.858979\n",
      "Epoche=222\n",
      "Action_times=15470\n",
      "Episolon=0.858105\n",
      "Epoche=223\n",
      "Action_times=15520\n",
      "Episolon=0.857681\n",
      "Epoche=224\n",
      "Action_times=15607\n",
      "Episolon=0.856944\n",
      "Epoche=225\n",
      "Action_times=15671\n",
      "Episolon=0.856402\n",
      "Epoche=226\n",
      "Action_times=15722\n",
      "Episolon=0.855971\n",
      "Epoche=227\n",
      "Action_times=15769\n",
      "Episolon=0.855573\n",
      "Epoche=228\n",
      "Action_times=15807\n",
      "Episolon=0.855252\n",
      "Epoche=229\n",
      "Action_times=15906\n",
      "Episolon=0.854416\n",
      "Epoche=230\n",
      "Action_times=15958\n",
      "Episolon=0.853977\n",
      "Epoche=231\n",
      "Action_times=16011\n",
      "Episolon=0.853530\n",
      "Epoche=232\n",
      "Action_times=16025\n",
      "Episolon=0.853411\n",
      "Epoche=233\n",
      "Action_times=16102\n",
      "Episolon=0.852762\n",
      "Epoche=234\n",
      "Action_times=16139\n",
      "Episolon=0.852451\n",
      "Epoche=235\n",
      "Action_times=16162\n",
      "Episolon=0.852257\n",
      "Epoche=236\n",
      "Action_times=16188\n",
      "Episolon=0.852038\n",
      "Epoche=237\n",
      "Action_times=16207\n",
      "Episolon=0.851878\n",
      "Epoche=238\n",
      "Action_times=16241\n",
      "Episolon=0.851592\n",
      "Epoche=239\n",
      "Action_times=16336\n",
      "Episolon=0.850793\n",
      "Epoche=240\n",
      "Action_times=16378\n",
      "Episolon=0.850439\n",
      "Epoche=241\n",
      "Action_times=16482\n",
      "Episolon=0.849566\n",
      "Epoche=242\n",
      "Action_times=16543\n",
      "Episolon=0.849054\n",
      "Epoche=243\n",
      "Action_times=16615\n",
      "Episolon=0.848450\n",
      "Epoche=244\n",
      "Action_times=16694\n",
      "Episolon=0.847788\n",
      "Epoche=245\n",
      "Action_times=16737\n",
      "Episolon=0.847428\n",
      "Epoche=246\n",
      "Action_times=16845\n",
      "Episolon=0.846524\n",
      "Epoche=247\n",
      "Action_times=16894\n",
      "Episolon=0.846114\n",
      "Epoche=248\n",
      "Action_times=16925\n",
      "Episolon=0.845855\n",
      "Epoche=249\n",
      "Action_times=16933\n",
      "Episolon=0.845788\n",
      "Epoche=250\n",
      "Action_times=16947\n",
      "Episolon=0.845671\n",
      "Epoche=251\n",
      "Action_times=17017\n",
      "Episolon=0.845086\n",
      "Epoche=252\n",
      "Action_times=17036\n",
      "Episolon=0.844928\n",
      "Epoche=253\n",
      "Action_times=17090\n",
      "Episolon=0.844477\n",
      "Epoche=254\n",
      "Action_times=17106\n",
      "Episolon=0.844343\n",
      "Epoche=255\n",
      "Action_times=17115\n",
      "Episolon=0.844268\n",
      "Epoche=256\n",
      "Action_times=17135\n",
      "Episolon=0.844101\n",
      "Epoche=257\n",
      "Action_times=17223\n",
      "Episolon=0.843368\n",
      "Epoche=258\n",
      "Action_times=17253\n",
      "Episolon=0.843118\n",
      "Epoche=259\n",
      "Action_times=17270\n",
      "Episolon=0.842976\n",
      "Epoche=260\n",
      "Action_times=17341\n",
      "Episolon=0.842385\n",
      "Epoche=261\n",
      "Action_times=17357\n",
      "Episolon=0.842252\n",
      "Epoche=262\n",
      "Action_times=17433\n",
      "Episolon=0.841619\n",
      "Epoche=263\n",
      "Action_times=17442\n",
      "Episolon=0.841545\n",
      "Epoche=264\n",
      "Action_times=17497\n",
      "Episolon=0.841087\n",
      "Epoche=265\n",
      "Action_times=17505\n",
      "Episolon=0.841021\n",
      "Epoche=266\n",
      "Action_times=17554\n",
      "Episolon=0.840614\n",
      "Epoche=267\n",
      "Action_times=17614\n",
      "Episolon=0.840116\n",
      "Epoche=268\n",
      "Action_times=17635\n",
      "Episolon=0.839941\n",
      "Epoche=269\n",
      "Action_times=17779\n",
      "Episolon=0.838747\n",
      "Epoche=270\n",
      "Action_times=17854\n",
      "Episolon=0.838126\n",
      "Epoche=271\n",
      "Action_times=17905\n",
      "Episolon=0.837703\n",
      "Epoche=272\n",
      "Action_times=17918\n",
      "Episolon=0.837596\n",
      "Epoche=273\n",
      "Action_times=17995\n",
      "Episolon=0.836959\n",
      "Epoche=274\n",
      "Action_times=18050\n",
      "Episolon=0.836504\n",
      "Epoche=275\n",
      "Action_times=18099\n",
      "Episolon=0.836099\n",
      "Epoche=276\n",
      "Action_times=18123\n",
      "Episolon=0.835901\n",
      "Epoche=277\n",
      "Action_times=18156\n",
      "Episolon=0.835629\n",
      "Epoche=278\n",
      "Action_times=18223\n",
      "Episolon=0.835076\n",
      "Epoche=279\n",
      "Action_times=18250\n",
      "Episolon=0.834853\n",
      "Epoche=280\n",
      "Action_times=18345\n",
      "Episolon=0.834070\n",
      "Epoche=281\n",
      "Action_times=18372\n",
      "Episolon=0.833847\n",
      "Epoche=282\n",
      "Action_times=18424\n",
      "Episolon=0.833419\n",
      "Epoche=283\n",
      "Action_times=18461\n",
      "Episolon=0.833114\n",
      "Epoche=284\n",
      "Action_times=18500\n",
      "Episolon=0.832793\n",
      "Epoche=285\n",
      "Action_times=18531\n",
      "Episolon=0.832538\n",
      "Epoche=286\n",
      "Action_times=18577\n",
      "Episolon=0.832160\n",
      "Epoche=287\n",
      "Action_times=18650\n",
      "Episolon=0.831560\n",
      "Epoche=288\n",
      "Action_times=18690\n",
      "Episolon=0.831231\n",
      "Epoche=289\n",
      "Action_times=18717\n",
      "Episolon=0.831010\n",
      "Epoche=290\n",
      "Action_times=18739\n",
      "Episolon=0.830829\n",
      "Epoche=291\n",
      "Action_times=18789\n",
      "Episolon=0.830419\n",
      "Epoche=292\n",
      "Action_times=18841\n",
      "Episolon=0.829992\n",
      "Epoche=293\n",
      "Action_times=18890\n",
      "Episolon=0.829591\n",
      "Epoche=294\n",
      "Action_times=19043\n",
      "Episolon=0.828338\n",
      "Epoche=295\n",
      "Action_times=19126\n",
      "Episolon=0.827659\n",
      "Epoche=296\n",
      "Action_times=19148\n",
      "Episolon=0.827479\n",
      "Epoche=297\n",
      "Action_times=19156\n",
      "Episolon=0.827413\n",
      "Epoche=298\n",
      "Action_times=19171\n",
      "Episolon=0.827291\n",
      "Epoche=299\n",
      "Action_times=19260\n",
      "Episolon=0.826564\n",
      "Epoche=300\n",
      "Action_times=19321\n",
      "Episolon=0.826066\n",
      "Epoche=301\n",
      "Action_times=19368\n",
      "Episolon=0.825682\n",
      "Epoche=302\n",
      "Action_times=19404\n",
      "Episolon=0.825389\n",
      "Epoche=303\n",
      "Action_times=19415\n",
      "Episolon=0.825299\n",
      "Epoche=304\n",
      "Action_times=19432\n",
      "Episolon=0.825160\n",
      "Epoche=305\n",
      "Action_times=19532\n",
      "Episolon=0.824346\n",
      "Epoche=306\n",
      "Action_times=19564\n",
      "Episolon=0.824085\n",
      "Epoche=307\n",
      "Action_times=19569\n",
      "Episolon=0.824044\n",
      "Epoche=308\n",
      "Action_times=19592\n",
      "Episolon=0.823857\n",
      "Epoche=309\n",
      "Action_times=19638\n",
      "Episolon=0.823483\n",
      "Epoche=310\n",
      "Action_times=19657\n",
      "Episolon=0.823328\n",
      "Epoche=311\n",
      "Action_times=19691\n",
      "Episolon=0.823052\n",
      "Epoche=312\n",
      "Action_times=19748\n",
      "Episolon=0.822589\n",
      "Epoche=313\n",
      "Action_times=19774\n",
      "Episolon=0.822377\n",
      "Epoche=314\n",
      "Action_times=19803\n",
      "Episolon=0.822142\n",
      "Epoche=315\n",
      "Action_times=19807\n",
      "Episolon=0.822109\n",
      "Epoche=316\n",
      "Action_times=19845\n",
      "Episolon=0.821801\n",
      "Epoche=317\n",
      "Action_times=19928\n",
      "Episolon=0.821127\n",
      "Epoche=318\n",
      "Action_times=19959\n",
      "Episolon=0.820876\n",
      "Epoche=319\n",
      "Action_times=19979\n",
      "Episolon=0.820714\n",
      "Epoche=320\n",
      "Action_times=20002\n",
      "Episolon=0.820527\n",
      "Epoche=321\n",
      "Action_times=20026\n",
      "Episolon=0.820333\n",
      "Epoche=322\n",
      "Action_times=20042\n",
      "Episolon=0.820203\n",
      "Epoche=323\n",
      "Action_times=20107\n",
      "Episolon=0.819677\n",
      "Epoche=324\n",
      "Action_times=20167\n",
      "Episolon=0.819191\n",
      "Epoche=325\n",
      "Action_times=20282\n",
      "Episolon=0.818261\n",
      "Epoche=326\n",
      "Action_times=20305\n",
      "Episolon=0.818075\n",
      "Epoche=327\n",
      "Action_times=20307\n",
      "Episolon=0.818059\n",
      "Epoche=328\n",
      "Action_times=20363\n",
      "Episolon=0.817607\n",
      "Epoche=329\n",
      "Action_times=20407\n",
      "Episolon=0.817251\n",
      "Epoche=330\n",
      "Action_times=20484\n",
      "Episolon=0.816630\n",
      "Epoche=331\n",
      "Action_times=20501\n",
      "Episolon=0.816493\n",
      "Epoche=332\n",
      "Action_times=20521\n",
      "Episolon=0.816331\n",
      "Epoche=333\n",
      "Action_times=20562\n",
      "Episolon=0.816001\n",
      "Epoche=334\n",
      "Action_times=20614\n",
      "Episolon=0.815582\n",
      "Epoche=335\n",
      "Action_times=20621\n",
      "Episolon=0.815526\n",
      "Epoche=336\n",
      "Action_times=20638\n",
      "Episolon=0.815389\n",
      "Epoche=337\n",
      "Action_times=20670\n",
      "Episolon=0.815131\n",
      "Epoche=338\n",
      "Action_times=20695\n",
      "Episolon=0.814930\n",
      "Epoche=339\n",
      "Action_times=20743\n",
      "Episolon=0.814543\n",
      "Epoche=340\n",
      "Action_times=20840\n",
      "Episolon=0.813763\n",
      "Epoche=341\n",
      "Action_times=20905\n",
      "Episolon=0.813241\n",
      "Epoche=342\n",
      "Action_times=21001\n",
      "Episolon=0.812470\n",
      "Epoche=343\n",
      "Action_times=21045\n",
      "Episolon=0.812117\n",
      "Epoche=344\n",
      "Action_times=21089\n",
      "Episolon=0.811765\n",
      "Epoche=345\n",
      "Action_times=21149\n",
      "Episolon=0.811284\n",
      "Epoche=346\n",
      "Action_times=21189\n",
      "Episolon=0.810963\n",
      "Epoche=347\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Action_times=21209\n",
      "Episolon=0.810803\n",
      "Epoche=348\n",
      "Action_times=21237\n",
      "Episolon=0.810579\n",
      "Epoche=349\n",
      "Action_times=21281\n",
      "Episolon=0.810227\n",
      "Epoche=350\n",
      "Action_times=21309\n",
      "Episolon=0.810003\n",
      "Epoche=351\n",
      "Action_times=21391\n",
      "Episolon=0.809347\n",
      "Epoche=352\n",
      "Action_times=21440\n",
      "Episolon=0.808955\n",
      "Epoche=353\n",
      "Action_times=21458\n",
      "Episolon=0.808811\n",
      "Epoche=354\n",
      "Action_times=21500\n",
      "Episolon=0.808476\n",
      "Epoche=355\n",
      "Action_times=21554\n",
      "Episolon=0.808045\n",
      "Epoche=356\n",
      "Action_times=21616\n",
      "Episolon=0.807550\n",
      "Epoche=357\n",
      "Action_times=21748\n",
      "Episolon=0.806498\n",
      "Epoche=358\n",
      "Action_times=21796\n",
      "Episolon=0.806116\n",
      "Epoche=359\n",
      "Action_times=21860\n",
      "Episolon=0.805607\n",
      "Epoche=360\n",
      "Action_times=21936\n",
      "Episolon=0.805002\n",
      "Epoche=361\n",
      "Action_times=21977\n",
      "Episolon=0.804676\n",
      "Epoche=362\n",
      "Action_times=22014\n",
      "Episolon=0.804382\n",
      "Epoche=363\n",
      "Action_times=22041\n",
      "Episolon=0.804168\n",
      "Epoche=364\n",
      "Action_times=22059\n",
      "Episolon=0.804025\n",
      "Epoche=365\n",
      "Action_times=22073\n",
      "Episolon=0.803914\n",
      "Epoche=366\n",
      "Action_times=22129\n",
      "Episolon=0.803469\n",
      "Epoche=367\n",
      "Action_times=22192\n",
      "Episolon=0.802970\n",
      "Epoche=368\n",
      "Action_times=22241\n",
      "Episolon=0.802581\n",
      "Epoche=369\n",
      "Action_times=22250\n",
      "Episolon=0.802510\n",
      "Epoche=370\n",
      "Action_times=22322\n",
      "Episolon=0.801939\n",
      "Epoche=371\n",
      "Action_times=22447\n",
      "Episolon=0.800950\n",
      "Epoche=372\n",
      "Action_times=22484\n",
      "Episolon=0.800658\n",
      "Epoche=373\n",
      "Action_times=22516\n",
      "Episolon=0.800405\n",
      "Epoche=374\n",
      "Action_times=22522\n",
      "Episolon=0.800357\n",
      "Epoche=375\n",
      "Action_times=22604\n",
      "Episolon=0.799709\n",
      "Epoche=376\n",
      "Action_times=22712\n",
      "Episolon=0.798857\n",
      "Epoche=377\n",
      "Action_times=22762\n",
      "Episolon=0.798463\n",
      "Epoche=378\n",
      "Action_times=22798\n",
      "Episolon=0.798179\n",
      "Epoche=379\n",
      "Action_times=22875\n",
      "Episolon=0.797572\n",
      "Epoche=380\n",
      "Action_times=22969\n",
      "Episolon=0.796832\n",
      "Epoche=381\n",
      "Action_times=23104\n",
      "Episolon=0.795771\n",
      "Epoche=382\n",
      "Action_times=23171\n",
      "Episolon=0.795244\n",
      "Epoche=383\n",
      "Action_times=23209\n",
      "Episolon=0.794946\n",
      "Epoche=384\n",
      "Action_times=23243\n",
      "Episolon=0.794679\n",
      "Epoche=385\n",
      "Action_times=23279\n",
      "Episolon=0.794397\n",
      "Epoche=386\n",
      "Action_times=23291\n",
      "Episolon=0.794303\n",
      "Epoche=387\n",
      "Action_times=23421\n",
      "Episolon=0.793284\n",
      "Epoche=388\n",
      "Action_times=23481\n",
      "Episolon=0.792814\n",
      "Epoche=389\n",
      "Action_times=23499\n",
      "Episolon=0.792673\n",
      "Epoche=390\n",
      "Action_times=23534\n",
      "Episolon=0.792399\n",
      "Epoche=391\n",
      "Action_times=23546\n",
      "Episolon=0.792305\n",
      "Epoche=392\n",
      "Action_times=23558\n",
      "Episolon=0.792211\n",
      "Epoche=393\n",
      "Action_times=23638\n",
      "Episolon=0.791586\n",
      "Epoche=394\n",
      "Action_times=23645\n",
      "Episolon=0.791531\n",
      "Epoche=395\n",
      "Action_times=23696\n",
      "Episolon=0.791133\n",
      "Epoche=396\n",
      "Action_times=23764\n",
      "Episolon=0.790602\n",
      "Epoche=397\n",
      "Action_times=23830\n",
      "Episolon=0.790087\n",
      "Epoche=398\n",
      "Action_times=23842\n",
      "Episolon=0.789993\n",
      "Epoche=399\n",
      "Action_times=23853\n",
      "Episolon=0.789907\n",
      "Epoche=400\n",
      "Action_times=23873\n",
      "Episolon=0.789751\n",
      "Epoche=401\n",
      "Action_times=23951\n",
      "Episolon=0.789143\n",
      "Epoche=402\n",
      "Action_times=23992\n",
      "Episolon=0.788824\n",
      "Epoche=403\n",
      "Action_times=24032\n",
      "Episolon=0.788512\n",
      "Epoche=404\n",
      "Action_times=24099\n",
      "Episolon=0.787991\n",
      "Epoche=405\n",
      "Action_times=24113\n",
      "Episolon=0.787882\n",
      "Epoche=406\n",
      "Action_times=24136\n",
      "Episolon=0.787703\n",
      "Epoche=407\n",
      "Action_times=24213\n",
      "Episolon=0.787105\n",
      "Epoche=408\n",
      "Action_times=24270\n",
      "Episolon=0.786662\n",
      "Epoche=409\n",
      "Action_times=24284\n",
      "Episolon=0.786553\n",
      "Epoche=410\n",
      "Action_times=24394\n",
      "Episolon=0.785699\n",
      "Epoche=411\n",
      "Action_times=24406\n",
      "Episolon=0.785606\n",
      "Epoche=412\n",
      "Action_times=24413\n",
      "Episolon=0.785552\n",
      "Epoche=413\n",
      "Action_times=24464\n",
      "Episolon=0.785156\n",
      "Epoche=414\n",
      "Action_times=24535\n",
      "Episolon=0.784606\n",
      "Epoche=415\n",
      "Action_times=24575\n",
      "Episolon=0.784297\n",
      "Epoche=416\n",
      "Action_times=24614\n",
      "Episolon=0.783995\n",
      "Epoche=417\n",
      "Action_times=24653\n",
      "Episolon=0.783693\n",
      "Epoche=418\n",
      "Action_times=24692\n",
      "Episolon=0.783391\n",
      "Epoche=419\n",
      "Action_times=24714\n",
      "Episolon=0.783221\n",
      "Epoche=420\n",
      "Action_times=24731\n",
      "Episolon=0.783090\n",
      "Epoche=421\n",
      "Action_times=24770\n",
      "Episolon=0.782788\n",
      "Epoche=422\n",
      "Action_times=24791\n",
      "Episolon=0.782626\n",
      "Epoche=423\n",
      "Action_times=24818\n",
      "Episolon=0.782417\n",
      "Epoche=424\n",
      "Action_times=24882\n",
      "Episolon=0.781923\n",
      "Epoche=425\n",
      "Action_times=24979\n",
      "Episolon=0.781175\n",
      "Epoche=426\n",
      "Action_times=25014\n",
      "Episolon=0.780905\n",
      "Epoche=427\n",
      "Action_times=25088\n",
      "Episolon=0.780335\n",
      "Epoche=428\n",
      "Action_times=25113\n",
      "Episolon=0.780142\n",
      "Epoche=429\n",
      "Action_times=25180\n",
      "Episolon=0.779626\n",
      "Epoche=430\n",
      "Action_times=25216\n",
      "Episolon=0.779349\n",
      "Epoche=431\n",
      "Action_times=25245\n",
      "Episolon=0.779126\n",
      "Epoche=432\n",
      "Action_times=25295\n",
      "Episolon=0.778742\n",
      "Epoche=433\n",
      "Action_times=25332\n",
      "Episolon=0.778457\n",
      "Epoche=434\n",
      "Action_times=25363\n",
      "Episolon=0.778219\n",
      "Epoche=435\n",
      "Action_times=25378\n",
      "Episolon=0.778104\n",
      "Epoche=436\n",
      "Action_times=25421\n",
      "Episolon=0.777774\n",
      "Epoche=437\n",
      "Action_times=25446\n",
      "Episolon=0.777582\n",
      "Epoche=438\n",
      "Action_times=25476\n",
      "Episolon=0.777351\n",
      "Epoche=439\n",
      "Action_times=25554\n",
      "Episolon=0.776753\n",
      "Epoche=440\n",
      "Action_times=25638\n",
      "Episolon=0.776109\n",
      "Epoche=441\n",
      "Action_times=25681\n",
      "Episolon=0.775780\n",
      "Epoche=442\n",
      "Action_times=25722\n",
      "Episolon=0.775466\n",
      "Epoche=443\n",
      "Action_times=25752\n",
      "Episolon=0.775237\n",
      "Epoche=444\n",
      "Action_times=25758\n",
      "Episolon=0.775191\n",
      "Epoche=445\n",
      "Action_times=25788\n",
      "Episolon=0.774961\n",
      "Epoche=446\n",
      "Action_times=25827\n",
      "Episolon=0.774663\n",
      "Epoche=447\n",
      "Action_times=25836\n",
      "Episolon=0.774594\n",
      "Epoche=448\n",
      "Action_times=25863\n",
      "Episolon=0.774388\n",
      "Epoche=449\n",
      "Action_times=25871\n",
      "Episolon=0.774326\n",
      "Epoche=450\n",
      "Action_times=25896\n",
      "Episolon=0.774135\n",
      "Epoche=451\n",
      "Action_times=25952\n",
      "Episolon=0.773708\n",
      "Epoche=452\n",
      "Action_times=25967\n",
      "Episolon=0.773593\n",
      "Epoche=453\n",
      "Action_times=25999\n",
      "Episolon=0.773349\n",
      "Epoche=454\n",
      "Action_times=26047\n",
      "Episolon=0.772982\n",
      "Epoche=455\n",
      "Action_times=26067\n",
      "Episolon=0.772830\n",
      "Epoche=456\n",
      "Action_times=26113\n",
      "Episolon=0.772479\n",
      "Epoche=457\n",
      "Action_times=26183\n",
      "Episolon=0.771945\n",
      "Epoche=458\n",
      "Action_times=26218\n",
      "Episolon=0.771679\n",
      "Epoche=459\n",
      "Action_times=26279\n",
      "Episolon=0.771214\n",
      "Epoche=460\n",
      "Action_times=26310\n",
      "Episolon=0.770978\n",
      "Epoche=461\n",
      "Action_times=26313\n",
      "Episolon=0.770956\n",
      "Epoche=462\n",
      "Action_times=26332\n",
      "Episolon=0.770811\n",
      "Epoche=463\n",
      "Action_times=26370\n",
      "Episolon=0.770522\n",
      "Epoche=464\n",
      "Action_times=26402\n",
      "Episolon=0.770279\n",
      "Epoche=465\n",
      "Action_times=26478\n",
      "Episolon=0.769701\n",
      "Epoche=466\n",
      "Action_times=26537\n",
      "Episolon=0.769253\n",
      "Epoche=467\n",
      "Action_times=26573\n",
      "Episolon=0.768980\n",
      "Epoche=468\n",
      "Action_times=26603\n",
      "Episolon=0.768752\n",
      "Epoche=469\n",
      "Action_times=26622\n",
      "Episolon=0.768608\n",
      "Epoche=470\n",
      "Action_times=26652\n",
      "Episolon=0.768380\n",
      "Epoche=471\n",
      "Action_times=26686\n",
      "Episolon=0.768122\n",
      "Epoche=472\n",
      "Action_times=26738\n",
      "Episolon=0.767728\n",
      "Epoche=473\n",
      "Action_times=26756\n",
      "Episolon=0.767592\n",
      "Epoche=474\n",
      "Action_times=26808\n",
      "Episolon=0.767198\n",
      "Epoche=475\n",
      "Action_times=26816\n",
      "Episolon=0.767138\n",
      "Epoche=476\n",
      "Action_times=26869\n",
      "Episolon=0.766736\n",
      "Epoche=477\n",
      "Action_times=26926\n",
      "Episolon=0.766305\n",
      "Epoche=478\n",
      "Action_times=27004\n",
      "Episolon=0.765715\n",
      "Epoche=479\n",
      "Action_times=27038\n",
      "Episolon=0.765459\n",
      "Epoche=480\n",
      "Action_times=27105\n",
      "Episolon=0.764953\n",
      "Epoche=481\n",
      "Action_times=27160\n",
      "Episolon=0.764537\n",
      "Epoche=482\n",
      "Action_times=27222\n",
      "Episolon=0.764070\n",
      "Epoche=483\n",
      "Action_times=27274\n",
      "Episolon=0.763678\n",
      "Epoche=484\n",
      "Action_times=27309\n",
      "Episolon=0.763414\n",
      "Epoche=485\n",
      "Action_times=27367\n",
      "Episolon=0.762977\n",
      "Epoche=486\n",
      "Action_times=27454\n",
      "Episolon=0.762322\n",
      "Epoche=487\n",
      "Action_times=27479\n",
      "Episolon=0.762134\n",
      "Epoche=488\n",
      "Action_times=27540\n",
      "Episolon=0.761676\n",
      "Epoche=489\n",
      "Action_times=27548\n",
      "Episolon=0.761616\n",
      "Epoche=490\n",
      "Action_times=27550\n",
      "Episolon=0.761601\n",
      "Epoche=491\n",
      "Action_times=27575\n",
      "Episolon=0.761413\n",
      "Epoche=492\n",
      "Action_times=27598\n",
      "Episolon=0.761240\n",
      "Epoche=493\n",
      "Action_times=27643\n",
      "Episolon=0.760902\n",
      "Epoche=494\n",
      "Action_times=27644\n",
      "Episolon=0.760894\n",
      "Epoche=495\n",
      "Action_times=27695\n",
      "Episolon=0.760511\n",
      "Epoche=496\n",
      "Action_times=27716\n",
      "Episolon=0.760354\n",
      "Epoche=497\n",
      "Action_times=27795\n",
      "Episolon=0.759761\n",
      "Epoche=498\n",
      "Action_times=27821\n",
      "Episolon=0.759566\n",
      "Epoche=499\n",
      "Action_times=27891\n",
      "Episolon=0.759042\n",
      "Epoche=500\n",
      "Action_times=27910\n",
      "Episolon=0.758900\n",
      "Epoche=501\n",
      "Action_times=27951\n",
      "Episolon=0.758593\n",
      "Epoche=502\n",
      "Action_times=28003\n",
      "Episolon=0.758203\n",
      "Epoche=503\n",
      "Action_times=28055\n",
      "Episolon=0.757814\n",
      "Epoche=504\n",
      "Action_times=28110\n",
      "Episolon=0.757403\n",
      "Epoche=505\n",
      "Action_times=28111\n",
      "Episolon=0.757396\n",
      "Epoche=506\n",
      "Action_times=28164\n",
      "Episolon=0.757000\n",
      "Epoche=507\n",
      "Action_times=28217\n",
      "Episolon=0.756604\n",
      "Epoche=508\n",
      "Action_times=28280\n",
      "Episolon=0.756134\n",
      "Epoche=509\n",
      "Action_times=28289\n",
      "Episolon=0.756067\n",
      "Epoche=510\n",
      "Action_times=28322\n",
      "Episolon=0.755820\n",
      "Epoche=511\n",
      "Action_times=28361\n",
      "Episolon=0.755530\n",
      "Epoche=512\n",
      "Action_times=28379\n",
      "Episolon=0.755395\n",
      "Epoche=513\n",
      "Action_times=28387\n",
      "Episolon=0.755336\n",
      "Epoche=514\n",
      "Action_times=28411\n",
      "Episolon=0.755157\n",
      "Epoche=515\n",
      "Action_times=28427\n",
      "Episolon=0.755038\n",
      "Epoche=516\n",
      "Action_times=28453\n",
      "Episolon=0.754844\n",
      "Epoche=517\n",
      "Action_times=28475\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Episolon=0.754680\n",
      "Epoche=518\n",
      "Action_times=28527\n",
      "Episolon=0.754293\n",
      "Epoche=519\n",
      "Action_times=28568\n",
      "Episolon=0.753988\n",
      "Epoche=520\n",
      "Action_times=28591\n",
      "Episolon=0.753817\n",
      "Epoche=521\n",
      "Action_times=28626\n",
      "Episolon=0.753557\n",
      "Epoche=522\n",
      "Action_times=28661\n",
      "Episolon=0.753296\n",
      "Epoche=523\n",
      "Action_times=28688\n",
      "Episolon=0.753096\n",
      "Epoche=524\n",
      "Action_times=28713\n",
      "Episolon=0.752910\n",
      "Epoche=525\n",
      "Action_times=28745\n",
      "Episolon=0.752672\n",
      "Epoche=526\n",
      "Action_times=28779\n",
      "Episolon=0.752420\n",
      "Epoche=527\n",
      "Action_times=28837\n",
      "Episolon=0.751989\n",
      "Epoche=528\n",
      "Action_times=28850\n",
      "Episolon=0.751893\n",
      "Epoche=529\n",
      "Action_times=28887\n",
      "Episolon=0.751618\n",
      "Epoche=530\n",
      "Action_times=28932\n",
      "Episolon=0.751285\n",
      "Epoche=531\n",
      "Action_times=28955\n",
      "Episolon=0.751114\n",
      "Epoche=532\n",
      "Action_times=28998\n",
      "Episolon=0.750796\n",
      "Epoche=533\n",
      "Action_times=29053\n",
      "Episolon=0.750388\n",
      "Epoche=534\n",
      "Action_times=29109\n",
      "Episolon=0.749974\n",
      "Epoche=535\n",
      "Action_times=29164\n",
      "Episolon=0.749567\n",
      "Epoche=536\n",
      "Action_times=29205\n",
      "Episolon=0.749264\n",
      "Epoche=537\n",
      "Action_times=29271\n",
      "Episolon=0.748776\n",
      "Epoche=538\n",
      "Action_times=29304\n",
      "Episolon=0.748532\n",
      "Epoche=539\n",
      "Action_times=29317\n",
      "Episolon=0.748436\n",
      "Epoche=540\n",
      "Action_times=29339\n",
      "Episolon=0.748274\n",
      "Epoche=541\n",
      "Action_times=29388\n",
      "Episolon=0.747912\n",
      "Epoche=542\n",
      "Action_times=29408\n",
      "Episolon=0.747765\n",
      "Epoche=543\n",
      "Action_times=29423\n",
      "Episolon=0.747654\n",
      "Epoche=544\n",
      "Action_times=29497\n",
      "Episolon=0.747108\n",
      "Epoche=545\n",
      "Action_times=29502\n",
      "Episolon=0.747072\n",
      "Epoche=546\n",
      "Action_times=29510\n",
      "Episolon=0.747013\n",
      "Epoche=547\n",
      "Action_times=29536\n",
      "Episolon=0.746821\n",
      "Epoche=548\n",
      "Action_times=29582\n",
      "Episolon=0.746482\n",
      "Epoche=549\n",
      "Action_times=29617\n",
      "Episolon=0.746224\n",
      "Epoche=550\n",
      "Action_times=29670\n",
      "Episolon=0.745834\n",
      "Epoche=551\n",
      "Action_times=29681\n",
      "Episolon=0.745753\n",
      "Epoche=552\n",
      "Action_times=29744\n",
      "Episolon=0.745290\n",
      "Epoche=553\n",
      "Action_times=29805\n",
      "Episolon=0.744842\n",
      "Epoche=554\n",
      "Action_times=29869\n",
      "Episolon=0.744371\n",
      "Epoche=555\n",
      "Action_times=29938\n",
      "Episolon=0.743865\n",
      "Epoche=556\n",
      "Action_times=29981\n",
      "Episolon=0.743549\n",
      "Epoche=557\n",
      "Action_times=30014\n",
      "Episolon=0.743307\n",
      "Epoche=558\n",
      "Action_times=30030\n",
      "Episolon=0.743190\n",
      "Epoche=559\n",
      "Action_times=30069\n",
      "Episolon=0.742904\n",
      "Epoche=560\n",
      "Action_times=30141\n",
      "Episolon=0.742377\n",
      "Epoche=561\n",
      "Action_times=30169\n",
      "Episolon=0.742172\n",
      "Epoche=562\n",
      "Action_times=30201\n",
      "Episolon=0.741937\n",
      "Epoche=563\n",
      "Action_times=30251\n",
      "Episolon=0.741571\n",
      "Epoche=564\n",
      "Action_times=30329\n",
      "Episolon=0.741001\n",
      "Epoche=565\n",
      "Action_times=30343\n",
      "Episolon=0.740899\n",
      "Epoche=566\n",
      "Action_times=30375\n",
      "Episolon=0.740665\n",
      "Epoche=567\n",
      "Action_times=30422\n",
      "Episolon=0.740322\n",
      "Epoche=568\n",
      "Action_times=30456\n",
      "Episolon=0.740073\n",
      "Epoche=569\n",
      "Action_times=30478\n",
      "Episolon=0.739913\n",
      "Epoche=570\n",
      "Action_times=30480\n",
      "Episolon=0.739898\n",
      "Epoche=571\n",
      "Action_times=30544\n",
      "Episolon=0.739431\n",
      "Epoche=572\n",
      "Action_times=30557\n",
      "Episolon=0.739336\n",
      "Epoche=573\n",
      "Action_times=30593\n",
      "Episolon=0.739074\n",
      "Epoche=574\n",
      "Action_times=30610\n",
      "Episolon=0.738950\n",
      "Epoche=575\n",
      "Action_times=30638\n",
      "Episolon=0.738746\n",
      "Epoche=576\n",
      "Action_times=30675\n",
      "Episolon=0.738476\n",
      "Epoche=577\n",
      "Action_times=30718\n",
      "Episolon=0.738163\n",
      "Epoche=578\n",
      "Action_times=30723\n",
      "Episolon=0.738127\n",
      "Epoche=579\n",
      "Action_times=30745\n",
      "Episolon=0.737966\n",
      "Epoche=580\n",
      "Action_times=30794\n",
      "Episolon=0.737610\n",
      "Epoche=581\n",
      "Action_times=30834\n",
      "Episolon=0.737319\n",
      "Epoche=582\n",
      "Action_times=30869\n",
      "Episolon=0.737064\n",
      "Epoche=583\n",
      "Action_times=30884\n",
      "Episolon=0.736955\n",
      "Epoche=584\n",
      "Action_times=30927\n",
      "Episolon=0.736643\n",
      "Epoche=585\n",
      "Action_times=30947\n",
      "Episolon=0.736497\n",
      "Epoche=586\n",
      "Action_times=30991\n",
      "Episolon=0.736178\n",
      "Epoche=587\n",
      "Action_times=31044\n",
      "Episolon=0.735793\n",
      "Epoche=588\n",
      "Action_times=31062\n",
      "Episolon=0.735662\n",
      "Epoche=589\n",
      "Action_times=31107\n",
      "Episolon=0.735336\n",
      "Epoche=590\n",
      "Action_times=31155\n",
      "Episolon=0.734988\n",
      "Epoche=591\n",
      "Action_times=31208\n",
      "Episolon=0.734604\n",
      "Epoche=592\n",
      "Action_times=31251\n",
      "Episolon=0.734292\n",
      "Epoche=593\n",
      "Action_times=31262\n",
      "Episolon=0.734213\n",
      "Epoche=594\n",
      "Action_times=31270\n",
      "Episolon=0.734155\n",
      "Epoche=595\n",
      "Action_times=31288\n",
      "Episolon=0.734024\n",
      "Epoche=596\n",
      "Action_times=31305\n",
      "Episolon=0.733901\n",
      "Epoche=597\n",
      "Action_times=31338\n",
      "Episolon=0.733662\n",
      "Epoche=598\n",
      "Action_times=31352\n",
      "Episolon=0.733561\n",
      "Epoche=599\n",
      "Action_times=31402\n",
      "Episolon=0.733199\n",
      "Epoche=600\n",
      "Action_times=31439\n",
      "Episolon=0.732932\n",
      "Epoche=601\n",
      "Action_times=31457\n",
      "Episolon=0.732802\n",
      "Epoche=602\n",
      "Action_times=31471\n",
      "Episolon=0.732701\n",
      "Epoche=603\n",
      "Action_times=31500\n",
      "Episolon=0.732491\n",
      "Epoche=604\n",
      "Action_times=31503\n",
      "Episolon=0.732469\n",
      "Epoche=605\n",
      "Action_times=31528\n",
      "Episolon=0.732289\n",
      "Epoche=606\n",
      "Action_times=31580\n",
      "Episolon=0.731913\n",
      "Epoche=607\n",
      "Action_times=31635\n",
      "Episolon=0.731516\n",
      "Epoche=608\n",
      "Action_times=31688\n",
      "Episolon=0.731134\n",
      "Epoche=609\n",
      "Action_times=31711\n",
      "Episolon=0.730968\n",
      "Epoche=610\n",
      "Action_times=31729\n",
      "Episolon=0.730838\n",
      "Epoche=611\n",
      "Action_times=31740\n",
      "Episolon=0.730759\n",
      "Epoche=612\n",
      "Action_times=31795\n",
      "Episolon=0.730363\n",
      "Epoche=613\n",
      "Action_times=31841\n",
      "Episolon=0.730031\n",
      "Epoche=614\n",
      "Action_times=31903\n",
      "Episolon=0.729585\n",
      "Epoche=615\n",
      "Action_times=31918\n",
      "Episolon=0.729477\n",
      "Epoche=616\n",
      "Action_times=31991\n",
      "Episolon=0.728952\n",
      "Epoche=617\n",
      "Action_times=32007\n",
      "Episolon=0.728837\n",
      "Epoche=618\n",
      "Action_times=32050\n",
      "Episolon=0.728528\n",
      "Epoche=619\n",
      "Action_times=32086\n",
      "Episolon=0.728270\n",
      "Epoche=620\n",
      "Action_times=32141\n",
      "Episolon=0.727875\n",
      "Epoche=621\n",
      "Action_times=32201\n",
      "Episolon=0.727444\n",
      "Epoche=622\n",
      "Action_times=32229\n",
      "Episolon=0.727243\n",
      "Epoche=623\n",
      "Action_times=32238\n",
      "Episolon=0.727179\n",
      "Epoche=624\n",
      "Action_times=32280\n",
      "Episolon=0.726877\n",
      "Epoche=625\n",
      "Action_times=32339\n",
      "Episolon=0.726455\n",
      "Epoche=626\n",
      "Action_times=32353\n",
      "Episolon=0.726354\n",
      "Epoche=627\n",
      "Action_times=32354\n",
      "Episolon=0.726347\n",
      "Epoche=628\n",
      "Action_times=32370\n",
      "Episolon=0.726233\n",
      "Epoche=629\n",
      "Action_times=32389\n",
      "Episolon=0.726097\n",
      "Epoche=630\n",
      "Action_times=32417\n",
      "Episolon=0.725896\n",
      "Epoche=631\n",
      "Action_times=32456\n",
      "Episolon=0.725617\n",
      "Epoche=632\n",
      "Action_times=32464\n",
      "Episolon=0.725560\n",
      "Epoche=633\n",
      "Action_times=32468\n",
      "Episolon=0.725531\n",
      "Epoche=634\n",
      "Action_times=32476\n",
      "Episolon=0.725474\n",
      "Epoche=635\n",
      "Action_times=32541\n",
      "Episolon=0.725009\n",
      "Epoche=636\n",
      "Action_times=32555\n",
      "Episolon=0.724909\n",
      "Epoche=637\n",
      "Action_times=32600\n",
      "Episolon=0.724587\n",
      "Epoche=638\n",
      "Action_times=32633\n",
      "Episolon=0.724351\n",
      "Epoche=639\n",
      "Action_times=32703\n",
      "Episolon=0.723851\n",
      "Epoche=640\n",
      "Action_times=32752\n",
      "Episolon=0.723502\n",
      "Epoche=641\n",
      "Action_times=32798\n",
      "Episolon=0.723174\n",
      "Epoche=642\n",
      "Action_times=32867\n",
      "Episolon=0.722682\n",
      "Epoche=643\n",
      "Action_times=32897\n",
      "Episolon=0.722468\n",
      "Epoche=644\n",
      "Action_times=32931\n",
      "Episolon=0.722226\n",
      "Epoche=645\n",
      "Action_times=32961\n",
      "Episolon=0.722012\n",
      "Epoche=646\n",
      "Action_times=33023\n",
      "Episolon=0.721571\n",
      "Epoche=647\n",
      "Action_times=33036\n",
      "Episolon=0.721478\n",
      "Epoche=648\n",
      "Action_times=33084\n",
      "Episolon=0.721137\n",
      "Epoche=649\n",
      "Action_times=33097\n",
      "Episolon=0.721044\n",
      "Epoche=650\n",
      "Action_times=33139\n",
      "Episolon=0.720746\n",
      "Epoche=651\n",
      "Action_times=33142\n",
      "Episolon=0.720725\n",
      "Epoche=652\n",
      "Action_times=33179\n",
      "Episolon=0.720462\n",
      "Epoche=653\n",
      "Action_times=33230\n",
      "Episolon=0.720099\n",
      "Epoche=654\n",
      "Action_times=33252\n",
      "Episolon=0.719943\n",
      "Epoche=655\n",
      "Action_times=33309\n",
      "Episolon=0.719539\n",
      "Epoche=656\n",
      "Action_times=33317\n",
      "Episolon=0.719482\n",
      "Epoche=657\n",
      "Action_times=33376\n",
      "Episolon=0.719063\n",
      "Epoche=658\n",
      "Action_times=33428\n",
      "Episolon=0.718695\n",
      "Epoche=659\n",
      "Action_times=33431\n",
      "Episolon=0.718674\n",
      "Epoche=660\n",
      "Action_times=33436\n",
      "Episolon=0.718638\n",
      "Epoche=661\n",
      "Action_times=33451\n",
      "Episolon=0.718532\n",
      "Epoche=662\n",
      "Action_times=33479\n",
      "Episolon=0.718333\n",
      "Epoche=663\n",
      "Action_times=33503\n",
      "Episolon=0.718163\n",
      "Epoche=664\n",
      "Action_times=33556\n",
      "Episolon=0.717788\n",
      "Epoche=665\n",
      "Action_times=33579\n",
      "Episolon=0.717625\n",
      "Epoche=666\n",
      "Action_times=33604\n",
      "Episolon=0.717449\n",
      "Epoche=667\n",
      "Action_times=33630\n",
      "Episolon=0.717265\n",
      "Epoche=668\n",
      "Action_times=33632\n",
      "Episolon=0.717251\n",
      "Epoche=669\n",
      "Action_times=33670\n",
      "Episolon=0.716982\n",
      "Epoche=670\n",
      "Action_times=33709\n",
      "Episolon=0.716706\n",
      "Epoche=671\n",
      "Action_times=33787\n",
      "Episolon=0.716155\n",
      "Epoche=672\n",
      "Action_times=33815\n",
      "Episolon=0.715957\n",
      "Epoche=673\n",
      "Action_times=33888\n",
      "Episolon=0.715442\n",
      "Epoche=674\n",
      "Action_times=33927\n",
      "Episolon=0.715167\n",
      "Epoche=675\n",
      "Action_times=33972\n",
      "Episolon=0.714850\n",
      "Epoche=676\n",
      "Action_times=34003\n",
      "Episolon=0.714631\n",
      "Epoche=677\n",
      "Action_times=34037\n",
      "Episolon=0.714392\n",
      "Epoche=678\n",
      "Action_times=34048\n",
      "Episolon=0.714314\n",
      "Epoche=679\n",
      "Action_times=34099\n",
      "Episolon=0.713955\n",
      "Epoche=680\n",
      "Action_times=34220\n",
      "Episolon=0.713104\n",
      "Epoche=681\n",
      "Action_times=34242\n",
      "Episolon=0.712949\n",
      "Epoche=682\n",
      "Action_times=34271\n",
      "Episolon=0.712746\n",
      "Epoche=683\n",
      "Action_times=34298\n",
      "Episolon=0.712556\n",
      "Epoche=684\n",
      "Action_times=34334\n",
      "Episolon=0.712303\n",
      "Epoche=685\n",
      "Action_times=34363\n",
      "Episolon=0.712099\n",
      "Epoche=686\n",
      "Action_times=34420\n",
      "Episolon=0.711699\n",
      "Epoche=687\n",
      "Action_times=34462\n",
      "Episolon=0.711405\n",
      "Epoche=688\n",
      "Action_times=34486\n",
      "Episolon=0.711236\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoche=689\n",
      "Action_times=34518\n",
      "Episolon=0.711012\n",
      "Epoche=690\n",
      "Action_times=34566\n",
      "Episolon=0.710676\n",
      "Epoche=691\n",
      "Action_times=34602\n",
      "Episolon=0.710423\n",
      "Epoche=692\n",
      "Action_times=34604\n",
      "Episolon=0.710409\n",
      "Epoche=693\n",
      "Action_times=34618\n",
      "Episolon=0.710311\n",
      "Epoche=694\n",
      "Action_times=34653\n",
      "Episolon=0.710066\n",
      "Epoche=695\n",
      "Action_times=34690\n",
      "Episolon=0.709807\n",
      "Epoche=696\n",
      "Action_times=34745\n",
      "Episolon=0.709422\n",
      "Epoche=697\n",
      "Action_times=34851\n",
      "Episolon=0.708681\n",
      "Epoche=698\n",
      "Action_times=34875\n",
      "Episolon=0.708514\n",
      "Epoche=699\n",
      "Action_times=34907\n",
      "Episolon=0.708290\n",
      "Epoche=700\n",
      "Action_times=34963\n",
      "Episolon=0.707899\n",
      "Epoche=701\n",
      "Action_times=35007\n",
      "Episolon=0.707592\n",
      "Epoche=702\n",
      "Action_times=35055\n",
      "Episolon=0.707258\n",
      "Epoche=703\n",
      "Action_times=35136\n",
      "Episolon=0.706693\n",
      "Epoche=704\n",
      "Action_times=35142\n",
      "Episolon=0.706651\n",
      "Epoche=705\n",
      "Action_times=35179\n",
      "Episolon=0.706394\n",
      "Epoche=706\n",
      "Action_times=35197\n",
      "Episolon=0.706268\n",
      "Epoche=707\n",
      "Action_times=35255\n",
      "Episolon=0.705864\n",
      "Epoche=708\n",
      "Action_times=35275\n",
      "Episolon=0.705725\n",
      "Epoche=709\n",
      "Action_times=35310\n",
      "Episolon=0.705482\n",
      "Epoche=710\n",
      "Action_times=35356\n",
      "Episolon=0.705162\n",
      "Epoche=711\n",
      "Action_times=35410\n",
      "Episolon=0.704787\n",
      "Epoche=712\n",
      "Action_times=35458\n",
      "Episolon=0.704453\n",
      "Epoche=713\n",
      "Action_times=35491\n",
      "Episolon=0.704224\n",
      "Epoche=714\n",
      "Action_times=35505\n",
      "Episolon=0.704127\n",
      "Epoche=715\n",
      "Action_times=35520\n",
      "Episolon=0.704023\n",
      "Epoche=716\n",
      "Action_times=35548\n",
      "Episolon=0.703829\n",
      "Epoche=717\n",
      "Action_times=35579\n",
      "Episolon=0.703614\n",
      "Epoche=718\n",
      "Action_times=35597\n",
      "Episolon=0.703489\n",
      "Epoche=719\n",
      "Action_times=35598\n",
      "Episolon=0.703482\n",
      "Epoche=720\n",
      "Action_times=35614\n",
      "Episolon=0.703371\n",
      "Epoche=721\n",
      "Action_times=35672\n",
      "Episolon=0.702969\n",
      "Epoche=722\n",
      "Action_times=35710\n",
      "Episolon=0.702705\n",
      "Epoche=723\n",
      "Action_times=35727\n",
      "Episolon=0.702588\n",
      "Epoche=724\n",
      "Action_times=35764\n",
      "Episolon=0.702332\n",
      "Epoche=725\n",
      "Action_times=35786\n",
      "Episolon=0.702179\n",
      "Epoche=726\n",
      "Action_times=35832\n",
      "Episolon=0.701861\n",
      "Epoche=727\n",
      "Action_times=35862\n",
      "Episolon=0.701653\n",
      "Epoche=728\n",
      "Action_times=35884\n",
      "Episolon=0.701501\n",
      "Epoche=729\n",
      "Action_times=35922\n",
      "Episolon=0.701239\n",
      "Epoche=730\n",
      "Action_times=35965\n",
      "Episolon=0.700941\n",
      "Epoche=731\n",
      "Action_times=35984\n",
      "Episolon=0.700810\n",
      "Epoche=732\n",
      "Action_times=36025\n",
      "Episolon=0.700527\n",
      "Epoche=733\n",
      "Action_times=36061\n",
      "Episolon=0.700278\n",
      "Epoche=734\n",
      "Action_times=36111\n",
      "Episolon=0.699933\n",
      "Epoche=735\n",
      "Action_times=36177\n",
      "Episolon=0.699478\n",
      "Epoche=736\n",
      "Action_times=36203\n",
      "Episolon=0.699299\n",
      "Epoche=737\n",
      "Action_times=36234\n",
      "Episolon=0.699085\n",
      "Epoche=738\n",
      "Action_times=36286\n",
      "Episolon=0.698727\n",
      "Epoche=739\n",
      "Action_times=36342\n",
      "Episolon=0.698341\n",
      "Epoche=740\n",
      "Action_times=36389\n",
      "Episolon=0.698018\n",
      "Epoche=741\n",
      "Action_times=36433\n",
      "Episolon=0.697715\n",
      "Epoche=742\n",
      "Action_times=36471\n",
      "Episolon=0.697454\n",
      "Epoche=743\n",
      "Action_times=36501\n",
      "Episolon=0.697248\n",
      "Epoche=744\n",
      "Action_times=36522\n",
      "Episolon=0.697104\n",
      "Epoche=745\n",
      "Action_times=36573\n",
      "Episolon=0.696753\n",
      "Epoche=746\n",
      "Action_times=36602\n",
      "Episolon=0.696554\n",
      "Epoche=747\n",
      "Action_times=36630\n",
      "Episolon=0.696362\n",
      "Epoche=748\n",
      "Action_times=36646\n",
      "Episolon=0.696252\n",
      "Epoche=749\n",
      "Action_times=36693\n",
      "Episolon=0.695930\n",
      "Epoche=750\n",
      "Action_times=36721\n",
      "Episolon=0.695738\n",
      "Epoche=751\n",
      "Action_times=36747\n",
      "Episolon=0.695559\n",
      "Epoche=752\n",
      "Action_times=36807\n",
      "Episolon=0.695148\n",
      "Epoche=753\n",
      "Action_times=36811\n",
      "Episolon=0.695121\n",
      "Epoche=754\n",
      "Action_times=36827\n",
      "Episolon=0.695011\n",
      "Epoche=755\n",
      "Action_times=36867\n",
      "Episolon=0.694737\n",
      "Epoche=756\n",
      "Action_times=36890\n",
      "Episolon=0.694580\n",
      "Epoche=757\n",
      "Action_times=36911\n",
      "Episolon=0.694436\n",
      "Epoche=758\n",
      "Action_times=36943\n",
      "Episolon=0.694217\n",
      "Epoche=759\n",
      "Action_times=36948\n",
      "Episolon=0.694183\n",
      "Epoche=760\n",
      "Action_times=36954\n",
      "Episolon=0.694142\n",
      "Epoche=761\n",
      "Action_times=36976\n",
      "Episolon=0.693991\n",
      "Epoche=762\n",
      "Action_times=37013\n",
      "Episolon=0.693738\n",
      "Epoche=763\n",
      "Action_times=37094\n",
      "Episolon=0.693184\n",
      "Epoche=764\n",
      "Action_times=37102\n",
      "Episolon=0.693130\n",
      "Epoche=765\n",
      "Action_times=37122\n",
      "Episolon=0.692993\n",
      "Epoche=766\n",
      "Action_times=37136\n",
      "Episolon=0.692898\n",
      "Epoche=767\n",
      "Action_times=37185\n",
      "Episolon=0.692563\n",
      "Epoche=768\n",
      "Action_times=37219\n",
      "Episolon=0.692331\n",
      "Epoche=769\n",
      "Action_times=37259\n",
      "Episolon=0.692058\n",
      "Epoche=770\n",
      "Action_times=37277\n",
      "Episolon=0.691935\n",
      "Epoche=771\n",
      "Action_times=37333\n",
      "Episolon=0.691554\n",
      "Epoche=772\n",
      "Action_times=37366\n",
      "Episolon=0.691329\n",
      "Epoche=773\n",
      "Action_times=37380\n",
      "Episolon=0.691233\n",
      "Epoche=774\n",
      "Action_times=37398\n",
      "Episolon=0.691111\n",
      "Epoche=775\n",
      "Action_times=37421\n",
      "Episolon=0.690954\n",
      "Epoche=776\n",
      "Action_times=37456\n",
      "Episolon=0.690716\n",
      "Epoche=777\n",
      "Action_times=37474\n",
      "Episolon=0.690593\n",
      "Epoche=778\n",
      "Action_times=37484\n",
      "Episolon=0.690525\n",
      "Epoche=779\n",
      "Action_times=37514\n",
      "Episolon=0.690321\n",
      "Epoche=780\n",
      "Action_times=37580\n",
      "Episolon=0.689872\n",
      "Epoche=781\n",
      "Action_times=37604\n",
      "Episolon=0.689709\n",
      "Epoche=782\n",
      "Action_times=37627\n",
      "Episolon=0.689553\n",
      "Epoche=783\n",
      "Action_times=37736\n",
      "Episolon=0.688812\n",
      "Epoche=784\n",
      "Action_times=37781\n",
      "Episolon=0.688507\n",
      "Epoche=785\n",
      "Action_times=37809\n",
      "Episolon=0.688317\n",
      "Epoche=786\n",
      "Action_times=37828\n",
      "Episolon=0.688188\n",
      "Epoche=787\n",
      "Action_times=37868\n",
      "Episolon=0.687917\n",
      "Epoche=788\n",
      "Action_times=37888\n",
      "Episolon=0.687781\n",
      "Epoche=789\n",
      "Action_times=37917\n",
      "Episolon=0.687585\n",
      "Epoche=790\n",
      "Action_times=37938\n",
      "Episolon=0.687443\n",
      "Epoche=791\n",
      "Action_times=37948\n",
      "Episolon=0.687375\n",
      "Epoche=792\n",
      "Action_times=38022\n",
      "Episolon=0.686874\n",
      "Epoche=793\n",
      "Action_times=38060\n",
      "Episolon=0.686617\n",
      "Epoche=794\n",
      "Action_times=38081\n",
      "Episolon=0.686475\n",
      "Epoche=795\n",
      "Action_times=38084\n",
      "Episolon=0.686454\n",
      "Epoche=796\n",
      "Action_times=38111\n",
      "Episolon=0.686272\n",
      "Epoche=797\n",
      "Action_times=38147\n",
      "Episolon=0.686028\n",
      "Epoche=798\n",
      "Action_times=38217\n",
      "Episolon=0.685555\n",
      "Epoche=799\n",
      "Action_times=38241\n",
      "Episolon=0.685393\n",
      "Epoche=800\n",
      "Action_times=38288\n",
      "Episolon=0.685076\n",
      "Epoche=801\n",
      "Action_times=38343\n",
      "Episolon=0.684705\n",
      "Epoche=802\n",
      "Action_times=38391\n",
      "Episolon=0.684381\n",
      "Epoche=803\n",
      "Action_times=38405\n",
      "Episolon=0.684286\n",
      "Epoche=804\n",
      "Action_times=38414\n",
      "Episolon=0.684226\n",
      "Epoche=805\n",
      "Action_times=38433\n",
      "Episolon=0.684098\n",
      "Epoche=806\n",
      "Action_times=38490\n",
      "Episolon=0.683713\n",
      "Epoche=807\n",
      "Action_times=38528\n",
      "Episolon=0.683458\n",
      "Epoche=808\n",
      "Action_times=38576\n",
      "Episolon=0.683134\n",
      "Epoche=809\n",
      "Action_times=38603\n",
      "Episolon=0.682953\n",
      "Epoche=810\n",
      "Action_times=38665\n",
      "Episolon=0.682536\n",
      "Epoche=811\n",
      "Action_times=38721\n",
      "Episolon=0.682159\n",
      "Epoche=812\n",
      "Action_times=38756\n",
      "Episolon=0.681924\n",
      "Epoche=813\n",
      "Action_times=38785\n",
      "Episolon=0.681729\n",
      "Epoche=814\n",
      "Action_times=38864\n",
      "Episolon=0.681199\n",
      "Epoche=815\n",
      "Action_times=38882\n",
      "Episolon=0.681078\n",
      "Epoche=816\n",
      "Action_times=38911\n",
      "Episolon=0.680883\n",
      "Epoche=817\n",
      "Action_times=38919\n",
      "Episolon=0.680829\n",
      "Epoche=818\n",
      "Action_times=38953\n",
      "Episolon=0.680601\n",
      "Epoche=819\n",
      "Action_times=38995\n",
      "Episolon=0.680320\n",
      "Epoche=820\n",
      "Action_times=39059\n",
      "Episolon=0.679891\n",
      "Epoche=821\n",
      "Action_times=39074\n",
      "Episolon=0.679790\n",
      "Epoche=822\n",
      "Action_times=39085\n",
      "Episolon=0.679717\n",
      "Epoche=823\n",
      "Action_times=39105\n",
      "Episolon=0.679583\n",
      "Epoche=824\n",
      "Action_times=39123\n",
      "Episolon=0.679462\n",
      "Epoche=825\n",
      "Action_times=39137\n",
      "Episolon=0.679369\n",
      "Epoche=826\n",
      "Action_times=39177\n",
      "Episolon=0.679101\n",
      "Epoche=827\n",
      "Action_times=39227\n",
      "Episolon=0.678766\n",
      "Epoche=828\n",
      "Action_times=39247\n",
      "Episolon=0.678633\n",
      "Epoche=829\n",
      "Action_times=39284\n",
      "Episolon=0.678385\n",
      "Epoche=830\n",
      "Action_times=39296\n",
      "Episolon=0.678305\n",
      "Epoche=831\n",
      "Action_times=39332\n",
      "Episolon=0.678065\n",
      "Epoche=832\n",
      "Action_times=39354\n",
      "Episolon=0.677918\n",
      "Epoche=833\n",
      "Action_times=39417\n",
      "Episolon=0.677497\n",
      "Epoche=834\n",
      "Action_times=39440\n",
      "Episolon=0.677344\n",
      "Epoche=835\n",
      "Action_times=39497\n",
      "Episolon=0.676963\n",
      "Epoche=836\n",
      "Action_times=39523\n",
      "Episolon=0.676790\n",
      "Epoche=837\n",
      "Action_times=39562\n",
      "Episolon=0.676530\n",
      "Epoche=838\n",
      "Action_times=39574\n",
      "Episolon=0.676450\n",
      "Epoche=839\n",
      "Action_times=39588\n",
      "Episolon=0.676357\n",
      "Epoche=840\n",
      "Action_times=39597\n",
      "Episolon=0.676297\n",
      "Epoche=841\n",
      "Action_times=39629\n",
      "Episolon=0.676083\n",
      "Epoche=842\n",
      "Action_times=39668\n",
      "Episolon=0.675824\n",
      "Epoche=843\n",
      "Action_times=39694\n",
      "Episolon=0.675651\n",
      "Epoche=844\n",
      "Action_times=39703\n",
      "Episolon=0.675591\n",
      "Epoche=845\n",
      "Action_times=39739\n",
      "Episolon=0.675351\n",
      "Epoche=846\n",
      "Action_times=39760\n",
      "Episolon=0.675211\n",
      "Epoche=847\n",
      "Action_times=39798\n",
      "Episolon=0.674959\n",
      "Epoche=848\n",
      "Action_times=39812\n",
      "Episolon=0.674866\n",
      "Epoche=849\n",
      "Action_times=39835\n",
      "Episolon=0.674713\n",
      "Epoche=850\n",
      "Action_times=39862\n",
      "Episolon=0.674533\n",
      "Epoche=851\n",
      "Action_times=39883\n",
      "Episolon=0.674394\n",
      "Epoche=852\n",
      "Action_times=39910\n",
      "Episolon=0.674214\n",
      "Epoche=853\n",
      "Action_times=39939\n",
      "Episolon=0.674022\n",
      "Epoche=854\n",
      "Action_times=39996\n",
      "Episolon=0.673643\n",
      "Epoche=855\n",
      "Action_times=40032\n",
      "Episolon=0.673405\n",
      "Epoche=856\n",
      "Action_times=40073\n",
      "Episolon=0.673133\n",
      "Epoche=857\n",
      "Action_times=40096\n",
      "Episolon=0.672980\n",
      "Epoche=858\n",
      "Action_times=40138\n",
      "Episolon=0.672702\n",
      "Epoche=859\n",
      "Action_times=40145\n",
      "Episolon=0.672655\n",
      "Epoche=860\n",
      "Action_times=40154\n",
      "Episolon=0.672596\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoche=861\n",
      "Action_times=40156\n",
      "Episolon=0.672582\n",
      "Epoche=862\n",
      "Action_times=40160\n",
      "Episolon=0.672556\n",
      "Epoche=863\n",
      "Action_times=40178\n",
      "Episolon=0.672437\n",
      "Epoche=864\n",
      "Action_times=40216\n",
      "Episolon=0.672185\n",
      "Epoche=865\n",
      "Action_times=40247\n",
      "Episolon=0.671980\n",
      "Epoche=866\n",
      "Action_times=40264\n",
      "Episolon=0.671867\n",
      "Epoche=867\n",
      "Action_times=40325\n",
      "Episolon=0.671464\n",
      "Epoche=868\n",
      "Action_times=40365\n",
      "Episolon=0.671199\n",
      "Epoche=869\n",
      "Action_times=40395\n",
      "Episolon=0.671001\n",
      "Epoche=870\n",
      "Action_times=40416\n",
      "Episolon=0.670862\n",
      "Epoche=871\n",
      "Action_times=40469\n",
      "Episolon=0.670512\n",
      "Epoche=872\n",
      "Action_times=40509\n",
      "Episolon=0.670248\n",
      "Epoche=873\n",
      "Action_times=40536\n",
      "Episolon=0.670069\n",
      "Epoche=874\n",
      "Action_times=40571\n",
      "Episolon=0.669838\n",
      "Epoche=875\n",
      "Action_times=40627\n",
      "Episolon=0.669469\n",
      "Epoche=876\n",
      "Action_times=40649\n",
      "Episolon=0.669324\n",
      "Epoche=877\n",
      "Action_times=40662\n",
      "Episolon=0.669238\n",
      "Epoche=878\n",
      "Action_times=40691\n",
      "Episolon=0.669047\n",
      "Epoche=879\n",
      "Action_times=40705\n",
      "Episolon=0.668955\n",
      "Epoche=880\n",
      "Action_times=40733\n",
      "Episolon=0.668770\n",
      "Epoche=881\n",
      "Action_times=40748\n",
      "Episolon=0.668672\n",
      "Epoche=882\n",
      "Action_times=40780\n",
      "Episolon=0.668461\n",
      "Epoche=883\n",
      "Action_times=40820\n",
      "Episolon=0.668197\n",
      "Epoche=884\n",
      "Action_times=40841\n",
      "Episolon=0.668059\n",
      "Epoche=885\n",
      "Action_times=40853\n",
      "Episolon=0.667980\n",
      "Epoche=886\n",
      "Action_times=40860\n",
      "Episolon=0.667934\n",
      "Epoche=887\n",
      "Action_times=40878\n",
      "Episolon=0.667816\n",
      "Epoche=888\n",
      "Action_times=40886\n",
      "Episolon=0.667763\n",
      "Epoche=889\n",
      "Action_times=40911\n",
      "Episolon=0.667599\n",
      "Epoche=890\n",
      "Action_times=40947\n",
      "Episolon=0.667362\n",
      "Epoche=891\n",
      "Action_times=40969\n",
      "Episolon=0.667217\n",
      "Epoche=892\n",
      "Action_times=41003\n",
      "Episolon=0.666994\n",
      "Epoche=893\n",
      "Action_times=41037\n",
      "Episolon=0.666771\n",
      "Epoche=894\n",
      "Action_times=41106\n",
      "Episolon=0.666318\n",
      "Epoche=895\n",
      "Action_times=41148\n",
      "Episolon=0.666042\n",
      "Epoche=896\n",
      "Action_times=41212\n",
      "Episolon=0.665622\n",
      "Epoche=897\n",
      "Action_times=41245\n",
      "Episolon=0.665406\n",
      "Epoche=898\n",
      "Action_times=41264\n",
      "Episolon=0.665282\n",
      "Epoche=899\n",
      "Action_times=41302\n",
      "Episolon=0.665033\n",
      "Epoche=900\n",
      "Action_times=41314\n",
      "Episolon=0.664954\n",
      "Epoche=901\n",
      "Action_times=41356\n",
      "Episolon=0.664679\n",
      "Epoche=902\n",
      "Action_times=41382\n",
      "Episolon=0.664509\n",
      "Epoche=903\n",
      "Action_times=41411\n",
      "Episolon=0.664319\n",
      "Epoche=904\n",
      "Action_times=41457\n",
      "Episolon=0.664018\n",
      "Epoche=905\n",
      "Action_times=41514\n",
      "Episolon=0.663645\n",
      "Epoche=906\n",
      "Action_times=41544\n",
      "Episolon=0.663449\n",
      "Epoche=907\n",
      "Action_times=41583\n",
      "Episolon=0.663195\n",
      "Epoche=908\n",
      "Action_times=41591\n",
      "Episolon=0.663142\n",
      "Epoche=909\n",
      "Action_times=41617\n",
      "Episolon=0.662972\n",
      "Epoche=910\n",
      "Action_times=41650\n",
      "Episolon=0.662757\n",
      "Epoche=911\n",
      "Action_times=41663\n",
      "Episolon=0.662672\n",
      "Epoche=912\n",
      "Action_times=41703\n",
      "Episolon=0.662411\n",
      "Epoche=913\n",
      "Action_times=41765\n",
      "Episolon=0.662007\n",
      "Epoche=914\n",
      "Action_times=41796\n",
      "Episolon=0.661805\n",
      "Epoche=915\n",
      "Action_times=41858\n",
      "Episolon=0.661401\n",
      "Epoche=916\n",
      "Action_times=41871\n",
      "Episolon=0.661316\n",
      "Epoche=917\n",
      "Action_times=41904\n",
      "Episolon=0.661101\n",
      "Epoche=918\n",
      "Action_times=41921\n",
      "Episolon=0.660990\n",
      "Epoche=919\n",
      "Action_times=41982\n",
      "Episolon=0.660593\n",
      "Epoche=920\n",
      "Action_times=42024\n",
      "Episolon=0.660320\n",
      "Epoche=921\n",
      "Action_times=42070\n",
      "Episolon=0.660021\n",
      "Epoche=922\n",
      "Action_times=42081\n",
      "Episolon=0.659950\n",
      "Epoche=923\n",
      "Action_times=42109\n",
      "Episolon=0.659768\n",
      "Epoche=924\n",
      "Action_times=42138\n",
      "Episolon=0.659579\n",
      "Epoche=925\n",
      "Action_times=42174\n",
      "Episolon=0.659346\n",
      "Epoche=926\n",
      "Action_times=42211\n",
      "Episolon=0.659105\n",
      "Epoche=927\n",
      "Action_times=42237\n",
      "Episolon=0.658937\n",
      "Epoche=928\n",
      "Action_times=42295\n",
      "Episolon=0.658560\n",
      "Epoche=929\n",
      "Action_times=42345\n",
      "Episolon=0.658236\n",
      "Epoche=930\n",
      "Action_times=42377\n",
      "Episolon=0.658029\n",
      "Epoche=931\n",
      "Action_times=42419\n",
      "Episolon=0.657757\n",
      "Epoche=932\n",
      "Action_times=42426\n",
      "Episolon=0.657711\n",
      "Epoche=933\n",
      "Action_times=42462\n",
      "Episolon=0.657478\n",
      "Epoche=934\n",
      "Action_times=42493\n",
      "Episolon=0.657277\n",
      "Epoche=935\n",
      "Action_times=42526\n",
      "Episolon=0.657064\n",
      "Epoche=936\n",
      "Action_times=42533\n",
      "Episolon=0.657019\n",
      "Epoche=937\n",
      "Action_times=42560\n",
      "Episolon=0.656844\n",
      "Epoche=938\n",
      "Action_times=42575\n",
      "Episolon=0.656747\n",
      "Epoche=939\n",
      "Action_times=42603\n",
      "Episolon=0.656566\n",
      "Epoche=940\n",
      "Action_times=42629\n",
      "Episolon=0.656398\n",
      "Epoche=941\n",
      "Action_times=42670\n",
      "Episolon=0.656133\n",
      "Epoche=942\n",
      "Action_times=42714\n",
      "Episolon=0.655848\n",
      "Epoche=943\n",
      "Action_times=42726\n",
      "Episolon=0.655771\n",
      "Epoche=944\n",
      "Action_times=42751\n",
      "Episolon=0.655610\n",
      "Epoche=945\n",
      "Action_times=42798\n",
      "Episolon=0.655306\n",
      "Epoche=946\n",
      "Action_times=42832\n",
      "Episolon=0.655087\n",
      "Epoche=947\n",
      "Action_times=42851\n",
      "Episolon=0.654964\n",
      "Epoche=948\n",
      "Action_times=42876\n",
      "Episolon=0.654803\n",
      "Epoche=949\n",
      "Action_times=42916\n",
      "Episolon=0.654545\n",
      "Epoche=950\n",
      "Action_times=42921\n",
      "Episolon=0.654513\n",
      "Epoche=951\n",
      "Action_times=42949\n",
      "Episolon=0.654333\n",
      "Epoche=952\n",
      "Action_times=42952\n",
      "Episolon=0.654313\n",
      "Epoche=953\n",
      "Action_times=42977\n",
      "Episolon=0.654152\n",
      "Epoche=954\n",
      "Action_times=43026\n",
      "Episolon=0.653837\n",
      "Epoche=955\n",
      "Action_times=43054\n",
      "Episolon=0.653656\n",
      "Epoche=956\n",
      "Action_times=43069\n",
      "Episolon=0.653560\n",
      "Epoche=957\n",
      "Action_times=43091\n",
      "Episolon=0.653418\n",
      "Epoche=958\n",
      "Action_times=43127\n",
      "Episolon=0.653187\n",
      "Epoche=959\n",
      "Action_times=43159\n",
      "Episolon=0.652981\n",
      "Epoche=960\n",
      "Action_times=43193\n",
      "Episolon=0.652762\n",
      "Epoche=961\n",
      "Action_times=43198\n",
      "Episolon=0.652730\n",
      "Epoche=962\n",
      "Action_times=43215\n",
      "Episolon=0.652621\n",
      "Epoche=963\n",
      "Action_times=43251\n",
      "Episolon=0.652390\n",
      "Epoche=964\n",
      "Action_times=43289\n",
      "Episolon=0.652146\n",
      "Epoche=965\n",
      "Action_times=43301\n",
      "Episolon=0.652068\n",
      "Epoche=966\n",
      "Action_times=43314\n",
      "Episolon=0.651985\n",
      "Epoche=967\n",
      "Action_times=43336\n",
      "Episolon=0.651844\n",
      "Epoche=968\n",
      "Action_times=43345\n",
      "Episolon=0.651786\n",
      "Epoche=969\n",
      "Action_times=43363\n",
      "Episolon=0.651671\n",
      "Epoche=970\n",
      "Action_times=43407\n",
      "Episolon=0.651388\n",
      "Epoche=971\n",
      "Action_times=43463\n",
      "Episolon=0.651029\n",
      "Epoche=972\n",
      "Action_times=43500\n",
      "Episolon=0.650792\n",
      "Epoche=973\n",
      "Action_times=43534\n",
      "Episolon=0.650574\n",
      "Epoche=974\n",
      "Action_times=43580\n",
      "Episolon=0.650280\n",
      "Epoche=975\n",
      "Action_times=43610\n",
      "Episolon=0.650088\n",
      "Epoche=976\n",
      "Action_times=43646\n",
      "Episolon=0.649857\n",
      "Epoche=977\n",
      "Action_times=43686\n",
      "Episolon=0.649601\n",
      "Epoche=978\n",
      "Action_times=43712\n",
      "Episolon=0.649435\n",
      "Epoche=979\n",
      "Action_times=43723\n",
      "Episolon=0.649365\n",
      "Epoche=980\n",
      "Action_times=43765\n",
      "Episolon=0.649096\n",
      "Epoche=981\n",
      "Action_times=43791\n",
      "Episolon=0.648930\n",
      "Epoche=982\n",
      "Action_times=43836\n",
      "Episolon=0.648643\n",
      "Epoche=983\n",
      "Action_times=43852\n",
      "Episolon=0.648540\n",
      "Epoche=984\n",
      "Action_times=43893\n",
      "Episolon=0.648279\n",
      "Epoche=985\n",
      "Action_times=43909\n",
      "Episolon=0.648177\n",
      "Epoche=986\n",
      "Action_times=43951\n",
      "Episolon=0.647909\n",
      "Epoche=987\n",
      "Action_times=43953\n",
      "Episolon=0.647896\n",
      "Epoche=988\n",
      "Action_times=43991\n",
      "Episolon=0.647653\n",
      "Epoche=989\n",
      "Action_times=44023\n",
      "Episolon=0.647449\n",
      "Epoche=990\n",
      "Action_times=44046\n",
      "Episolon=0.647303\n",
      "Epoche=991\n",
      "Action_times=44061\n",
      "Episolon=0.647207\n",
      "Epoche=992\n",
      "Action_times=44068\n",
      "Episolon=0.647163\n",
      "Epoche=993\n",
      "Action_times=44107\n",
      "Episolon=0.646914\n",
      "Epoche=994\n",
      "Action_times=44155\n",
      "Episolon=0.646609\n",
      "Epoche=995\n",
      "Action_times=44177\n",
      "Episolon=0.646469\n",
      "Epoche=996\n",
      "Action_times=44191\n",
      "Episolon=0.646379\n",
      "Epoche=997\n",
      "Action_times=44223\n",
      "Episolon=0.646176\n",
      "Epoche=998\n",
      "Action_times=44237\n",
      "Episolon=0.646087\n",
      "Epoche=999\n",
      "Action_times=44326\n",
      "Episolon=0.645521\n"
     ]
    }
   ],
   "source": [
    "def Run_Function():\n",
    "    Q_Table=Initial_Q_Table(TotalNumberOfMesh,Actions)\n",
    "    Action_times=0 \n",
    "    Epoche=0\n",
    "    while Epoche<1000:\n",
    "        if(np.random.rand()<0.4):\n",
    "            Current_state=1\n",
    "        else:\n",
    "            Current_state=np.random.randint(low=5, high=100)\n",
    "        Is_end=False\n",
    "        while not Is_end:\n",
    "            Action=Select_action(Current_state,Q_Table,Action_times)\n",
    "            Next_state,Reward=Output_next_state(Current_state,Action)\n",
    "            Q_table_state_action=Q_Table.loc[[Current_state],[Action]]\n",
    "            if Next_state!='End':\n",
    "                Next_state_column=np.array((Q_Table.loc[[Next_state],:]).T)\n",
    "                Q_target=Reward+Beta*Next_state_column.max() #Next_S状态中的可选动作中的最大Q值动作\n",
    "                Action_times+=1\n",
    "            else:\n",
    "                print('Epoche=%d'%Epoche)\n",
    "                print('Action_times=%d'%Action_times)\n",
    "                print('Episolon=%f'%(Epsilon_stop+(Epsilon_start-Epsilon_stop)*np.exp(-Decay_rate*Action_times)))\n",
    "                Q_target=Reward\n",
    "                Action_times+=1\n",
    "                Is_end=True \n",
    "            Q_Table.loc[[Current_state],[Action]]+=Alpha*(Q_target-Q_table_state_action)\n",
    "            Current_state=Next_state\n",
    "        Epoche+=1\n",
    " \n",
    "    return Q_Table\n",
    "\n",
    "Q__Table=Run_Function()\n",
    "Q__Table\n",
    "Final_Q_Table=np.array(Q__Table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "Current_state=1\n",
      "Action=Up_four\n",
      "Next_state=11\n",
      "--------------------------------------------------\n",
      "Current_state=11\n",
      "Action=Right_four\n",
      "Next_state=12\n",
      "--------------------------------------------------\n",
      "Current_state=12\n",
      "Action=Up_six\n",
      "Next_state=22\n",
      "--------------------------------------------------\n",
      "Current_state=22\n",
      "Action=Up_three\n",
      "Next_state=32\n",
      "--------------------------------------------------\n",
      "Current_state=32\n",
      "Action=Up_four\n",
      "Next_state=42\n",
      "--------------------------------------------------\n",
      "Current_state=42\n",
      "Action=Up_four\n",
      "Next_state=52\n",
      "--------------------------------------------------\n",
      "Current_state=52\n",
      "Action=Up_four\n",
      "Next_state=62\n",
      "--------------------------------------------------\n",
      "Current_state=62\n",
      "Action=Up_six\n",
      "Next_state=72\n",
      "--------------------------------------------------\n",
      "Current_state=72\n",
      "Action=Up_six\n",
      "Next_state=82\n",
      "--------------------------------------------------\n",
      "Current_state=82\n",
      "Action=Up_six\n",
      "Next_state=92\n",
      "--------------------------------------------------\n",
      "Current_state=92\n",
      "Action=Right_four\n",
      "Next_state=93\n",
      "--------------------------------------------------\n",
      "Current_state=93\n",
      "Action=Right_three\n",
      "Next_state=94\n",
      "--------------------------------------------------\n",
      "Current_state=94\n",
      "Action=Right_three\n",
      "Next_state=95\n",
      "--------------------------------------------------\n",
      "Current_state=95\n",
      "Action=Right_three\n",
      "Next_state=96\n",
      "--------------------------------------------------\n",
      "Current_state=96\n",
      "Action=Right_three\n",
      "Next_state=97\n",
      "--------------------------------------------------\n",
      "Current_state=97\n",
      "Action=Right_three\n",
      "Next_state=98\n",
      "--------------------------------------------------\n",
      "Current_state=98\n",
      "Action=Right_three\n",
      "Next_state=99\n",
      "--------------------------------------------------\n",
      "Current_state=99\n",
      "Action=Right_six\n",
      "Next_state=100\n"
     ]
    }
   ],
   "source": [
    "def Plot_Final_Graph(Q_TABLE):\n",
    "    fig=plt.figure()\n",
    "    ax=fig.gca()\n",
    "    ax.set(xlim=[0, LengthOfMesh], ylim=[0, LengthOfMesh])\n",
    "    ax.set_xticks(np.arange(0,(LengthOfMesh+1)))\n",
    "    ax.set_yticks(np.arange(0,(LengthOfMesh+1)))\n",
    "    plt.xlabel('x',size=30)\n",
    "    plt.ylabel('y',size=30)\n",
    "    fig.suptitle('Result of grid map (three types of obstacles)', fontsize=30)\n",
    "    plt.grid()\n",
    "    Current_statestart=1\n",
    "    Current_state=Current_statestart\n",
    "    Action_times=1\n",
    "    while(Current_state!=TotalNumberOfMesh):\n",
    "        Action=np.argmax((Q_TABLE[Current_state-1]))\n",
    "        print('--------------------------------------------------')\n",
    "        print('Current_state=%d'%Current_state)\n",
    "        print('Action=%s'%(Actions[Action]))       \n",
    "        if(Action in [0,1,2]):\n",
    "             Next_state=Current_state-1\n",
    "        elif(Action in [3,4,5]):\n",
    "             Next_state=Current_state+1\n",
    "        else:\n",
    "             Next_state=Current_state+LengthOfMesh\n",
    "        print('Next_state=%d'%Next_state)\n",
    "        Action_times+=1\n",
    "        x_axis_current_state,y_axis_current_state=Caculate_coordinate(Current_state)\n",
    "        x_axis_next_state,y_axis_next_state=Caculate_coordinate(Next_state)\n",
    "        if(Action_times==0):\n",
    "            plt.scatter(x_axis_current_state,y_axis_current_state)  \n",
    "        else:\n",
    "            plt.scatter(x_axis_current_state,y_axis_current_state) \n",
    "            plt.scatter(x_axis_next_state,y_axis_next_state)\n",
    "            plt.plot([x_axis_current_state,x_axis_next_state],[y_axis_current_state,y_axis_next_state])\n",
    "        for J in np.arange(len(Obstacle)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle[J]))[0],(Caculate_coordinate(Obstacle[J]))[1],marker=\"x\",color='k',s=300)\n",
    "        for J in np.arange(len(Obstacle_six)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle_six[J]))[0],(Caculate_coordinate(Obstacle_six[J]))[1],marker=\"H\",color='k',s=300)\n",
    "        for J in np.arange(len(Obstacle_four)):\n",
    "            plt.scatter((Caculate_coordinate(Obstacle_four[J]))[0],(Caculate_coordinate(Obstacle_four[J]))[1],marker=\"s\",color='k',s=300)\n",
    "        Current_state=Next_state\n",
    "    plt.show()\n",
    "Plot_Final_Graph(Final_Q_Table)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Plot_Final_Graph(Final_Q_Table)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
